supertable 2.2.5__tar.gz → 2.2.7__tar.gz

{supertable-2.2.5/supertable.egg-info → supertable-2.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supertable
-Version: 2.2.5
+Version: 2.2.7
 Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
 Author: Levente Kupas
 Author-email: Levente Kupas <lkupas@kladnasoft.com>

{supertable-2.2.5 → supertable-2.2.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "supertable"
-version = "2.2.5"
+version = "2.2.7"
 description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
 readme = "README.md"
 requires-python = ">=3.10"

{supertable-2.2.5 → supertable-2.2.7}/setup.py

@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
 
 setup(
     name="supertable",
-    version="2.2.5",
+    version="2.2.7",
     description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
     long_description=long_description,
     long_description_content_type="text/markdown",

{supertable-2.2.5 → supertable-2.2.7}/supertable/__init__.py

@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
 project documentation for the full API surface.
 """
 
-__version__ = "2.2.5"
+__version__ = "2.2.7"
 
 # Re-export the core public surface so users can do ``from supertable import …``
 # instead of remembering submodule paths.

{supertable-2.2.5 → supertable-2.2.7}/supertable/data_reader.py

@@ -44,10 +44,20 @@ class DataReader:
       - Execution to Executor (DuckDB/Spark)
     """
 
-    def __init__(self, super_name: str, organization: str, query: str):
+    def __init__(
+        self,
+        super_name: str,
+        organization: str,
+        query: str,
+        source: str = "sdk",
+    ):
         self.super_name = super_name
         self.organization = organization
         self.query = query
+        # Query origin surfaced in the reads monitoring tab. "sdk" is the
+        # default for direct SDK callers; the API/OData/MCP entry points
+        # pass "api"/"odata"/"mcp" so each query records where it came from.
+        self.source = source
 
         self.storage: StorageInterface = get_storage()
 
@@ -164,6 +174,9 @@ class DataReader:
                 current_meta_path="redis://meta/root",
                 query=parser.original_query,
             )
+            # Stamp the call origin so plan_extender records it on the read
+            # monitoring entry (defaults to "api" downstream if unset).
+            self.query_plan_manager.source_type = self.source
             self._log_ctx = f"[qid={self.query_plan_manager.query_id} qh={self.query_plan_manager.query_hash}] "
             self.query_plan_manager.original_table = ", ".join(t.simple_name for t in physical_tables) if physical_tables else ""
 
@@ -311,16 +324,25 @@ def query_sql(
         limit: int,
         engine: Any,
         role_name: str,
+        source: str = "sdk",
+        out: Optional[Dict[str, Any]] = None,
 ) -> Tuple[List[str], List[List[Any]], List[Dict[str, Any]]]:
     """
     Execute SQL query and return results in the format expected by MCP server.
     Returns: (columns, rows, columns_meta)
+
+    ``source`` tags the query origin on the read monitoring entry
+    (defaults to "sdk"; the MCP server passes "mcp"). When an ``out``
+    dict is supplied it is populated with ``query_id``/``query_hash`` so
+    the caller can correlate its own audit log to this read record.
     """
     # Safety guard: ensure a LIMIT is present so unbounded queries don't
     # overwhelm the MCP response payload.
     sql = _ensure_sql_limit(sql, default_limit=limit)
 
-    reader = DataReader(organization=organization, super_name=super_name, query=sql)
+    reader = DataReader(
+        organization=organization, super_name=super_name, query=sql, source=source,
+    )
 
     # Execute the query
     result_df, status, message = reader.execute(
@@ -329,6 +351,15 @@ def query_sql(
         with_scan=False,
     )
 
+    # Expose the query identity so the caller (e.g. the MCP audit log) can
+    # link back to this read's monitoring entry. Populated even on error,
+    # since the QueryPlanManager is created before execution.
+    if out is not None:
+        qpm = reader.query_plan_manager
+        if qpm is not None:
+            out["query_id"] = qpm.query_id
+            out["query_hash"] = qpm.query_hash
+
     if status == Status.ERROR:
         raise RuntimeError(f"Query execution failed: {message}")
 

{supertable-2.2.5 → supertable-2.2.7}/supertable/engine/engine_common.py

@@ -14,6 +14,7 @@ from sqlglot import exp
 from supertable.config.defaults import logger
 from supertable.config.settings import settings
 from supertable.config.homedir import get_app_home
+from supertable.engine.engine_config import normalize_memory_size
 
 
 # =========================================================
@@ -578,8 +579,18 @@ def init_connection(
     # Resolve memory limit.
     # Single env var SUPERTABLE_DUCKDB_MEMORY_LIMIT controls both executors.
     # The `memory_limit` argument is the caller's fallback when the env var is absent.
-    effective_memory_limit = settings.SUPERTABLE_DUCKDB_MEMORY_LIMIT or memory_limit
-    con.execute(f"PRAGMA memory_limit='{effective_memory_limit}';")
+    # normalize_memory_size guarantees a unit-suffixed value so DuckDB's parser
+    # never rejects a bare number (e.g. a UI-supplied "2" -> "2GB").
+    effective_memory_limit = normalize_memory_size(
+        settings.SUPERTABLE_DUCKDB_MEMORY_LIMIT or memory_limit, default="1GB"
+    )
+    try:
+        con.execute(f"PRAGMA memory_limit='{effective_memory_limit}';")
+    except Exception as e:
+        logger.warning(
+            f"[duckdb.init] memory_limit='{effective_memory_limit}' rejected: {e}; "
+            f"keeping DuckDB default"
+        )
 
     # Absolute temp path is required for DuckDB to actually spill to disk.
     # Prefer a path rooted under the app home (~/supertable), which is
@@ -647,11 +658,14 @@ def apply_runtime_pragmas(con: duckdb.DuckDBPyConnection, cfg) -> None:
     if cfg is None:
         return
 
-    memory_limit = (cfg.duckdb_memory_limit or "").strip() or "1GB"
+    # normalize_memory_size guarantees a unit (UI sends a bare "2" -> "2GB");
+    # a rejected value is logged rather than silently swallowed so a bad
+    # config can never quietly leave the connection at the wrong limit.
+    memory_limit = normalize_memory_size(cfg.duckdb_memory_limit, default="1GB")
     try:
         con.execute(f"PRAGMA memory_limit='{memory_limit}';")
-    except Exception:
-        pass
+    except Exception as e:
+        logger.warning(f"[duckdb.pragma] memory_limit='{memory_limit}' rejected: {e}")
 
     # Explicit thread count wins; otherwise derive from the live memory limit
     # and IO multiplier (same formula as init_connection).
@@ -671,13 +685,32 @@ def apply_runtime_pragmas(con: duckdb.DuckDBPyConnection, cfg) -> None:
         except Exception:
             pass
 
-    cache_size = (cfg.duckdb_external_cache_size or "").strip()
+    cache_size = normalize_memory_size(cfg.duckdb_external_cache_size, default="")
     if cache_size:
         try:
             con.execute("SET enable_external_file_cache=true;")
-            con.execute(f"SET external_file_cache_max_size='{sanitize_sql_string(cache_size)}';")
-        except Exception:
-            pass
+        except Exception as e:
+            logger.warning(f"[duckdb.pragma] enable_external_file_cache failed: {e}")
+        # The size cap is version-dependent: ``external_file_cache_max_size``
+        # does not exist on every DuckDB build (e.g. 1.5.x).  Probe the
+        # settings catalog and only set it when present so we neither raise
+        # nor silently swallow a genuinely unsupported setting.
+        try:
+            has_cap = bool(con.execute(
+                "SELECT 1 FROM duckdb_settings() "
+                "WHERE name='external_file_cache_max_size'"
+            ).fetchone())
+            if has_cap:
+                con.execute(
+                    f"SET external_file_cache_max_size='{sanitize_sql_string(cache_size)}';"
+                )
+            else:
+                logger.debug(
+                    "[duckdb.pragma] external_file_cache_max_size unsupported on this "
+                    "DuckDB version; cache enabled without a size cap"
+                )
+        except Exception as e:
+            logger.warning(f"[duckdb.pragma] external_file_cache_max_size failed: {e}")
 
 
 # =========================================================

supertable-2.2.7/supertable/engine/engine_config.py

@@ -0,0 +1,271 @@
+# route: supertable.engine.engine_config
+"""
+Engine runtime configuration — single source of truth for resolution.
+
+Engine config is stored once per organization at **system scope**
+(``supertable:{org}:system:engine:duckdb``).  It is global to the org, not
+per-supertable, and is resolved **live on every query** (no caching) so that
+changes made in the UI take effect on the next query without a restart or a
+connection reset.
+
+Two kinds of settings live in that document:
+
+  * **Shared auto-pick thresholds** (one value, used by the router):
+    ``engine_lite_max_bytes``, ``engine_spark_min_bytes``,
+    ``engine_freshness_sec``.  These describe the *boundaries* between engines
+    so they cannot be per-engine.
+  * **Per-engine DuckDB runtime pragmas** (separate values for Lite and Pro):
+    ``duckdb_memory_limit``, ``duckdb_io_multiplier``, ``duckdb_threads``,
+    ``duckdb_http_timeout``, ``duckdb_external_cache_size``.  Each DuckDB engine
+    carries its own copy under a ``"lite"`` / ``"pro"`` section, so tuning Lite
+    never changes Pro and vice-versa.
+
+Stored document shape::
+
+    {
+      "engine_lite_max_bytes": "104857600",
+      "engine_spark_min_bytes": "10737418240",
+      "engine_freshness_sec":   "300",
+      "lite": {"duckdb_memory_limit": "1GB", ...},
+      "pro":  {"duckdb_memory_limit": "8GB", ...},
+      "modified_ms": 1750000000000
+    }
+
+Resolution precedence, per field::
+
+    Redis (org system config)  →  environment variable  →  built-in default
+
+The environment variable is a single global fallback shared by both engines;
+per-engine differentiation therefore only happens once a value is stored in
+Redis (via the UI).
+
+Entry points (all share that precedence so engine and UI never disagree):
+
+  * :func:`resolve_engine_configs`           → ``{"lite": cfg, "pro": cfg}`` in a
+                                                single Redis read; used by the
+                                                executor.
+  * :func:`resolve_engine_config`            → typed :class:`EngineRuntimeConfig`
+                                                for one engine (convenience).
+  * :func:`resolve_engine_config_provenance` → ``{shared, lite, pro,
+                                                modified_ms}`` per-field
+                                                ``{value, source, env_var,
+                                                default}`` for the API / UI.
+
+Built-in defaults mirror the :mod:`supertable.config.settings` dataclass
+defaults, so behaviour is identical whether a value flows through ``settings``
+(no Redis override) or through this resolver.
+"""
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import dataclass
+from typing import Any, Dict, Optional, Tuple
+
+
+# Optional decimal number followed by an optional byte unit (decimal KB/MB/GB/TB
+# or binary KiB/MiB/GiB/TiB), case-insensitive, whitespace-tolerant.
+_MEM_SIZE_RE = re.compile(r"^(\d+(?:\.\d+)?)\s*([KMGT]i?B)?$", re.IGNORECASE)
+
+
+def normalize_memory_size(value: Any, default: str = "1GB", *, bare_unit: str = "GB") -> str:
+    """Coerce a memory-size value into a DuckDB-valid, unit-suffixed string.
+
+    The engine UI collects a bare integer (interpreted as ``bare_unit`` — GB),
+    while env vars / older Redis docs may already carry a unit.  DuckDB's
+    ``PRAGMA memory_limit`` parser rejects a unit-less number
+    (``Unknown unit for memory: ''``), so this guarantees a value the parser
+    accepts before it is ever sent:
+
+      * bare number            -> ``"<n>GB"``       ("2"    -> "2GB")
+      * number + unit          -> canonical casing  ("2gib" -> "2GiB")
+      * empty / None           -> ``default``       (""/None -> default)
+      * non-positive / garbage -> ``default``       ("0", "x" -> default)
+
+    ``default`` is returned verbatim, so pass ``""`` when "unset" is the
+    desired fallback (e.g. the external file cache, where empty == disabled).
+    """
+    s = ("" if value is None else str(value)).strip()
+    if not s:
+        return default
+    m = _MEM_SIZE_RE.match(s)
+    if not m:
+        return default
+    num, unit = m.group(1), m.group(2)
+    try:
+        if float(num) <= 0:
+            return default
+    except ValueError:
+        return default
+    if unit:
+        u = unit.upper()
+        unit = (u[0] + "iB") if u.endswith("IB") else u
+    else:
+        unit = bare_unit
+    return f"{num}{unit}"
+
+
+# Shared auto-pick thresholds: key → (env var, built-in default as string).
+_SHARED_SPEC: Dict[str, Tuple[str, str]] = {
+    "engine_lite_max_bytes":  ("SUPERTABLE_ENGINE_LITE_MAX_BYTES",  str(100 * 1024 * 1024)),
+    "engine_spark_min_bytes": ("SUPERTABLE_ENGINE_SPARK_MIN_BYTES", str(10 * 1024 * 1024 * 1024)),
+    "engine_freshness_sec":   ("SUPERTABLE_ENGINE_FRESHNESS_SEC",   "300"),
+}
+
+# Per-engine DuckDB runtime pragmas: key → (env var, built-in default as string).
+# Defaults mirror the settings dataclass defaults in supertable/config/settings.py.
+_DUCKDB_SPEC: Dict[str, Tuple[str, str]] = {
+    "duckdb_memory_limit":        ("SUPERTABLE_DUCKDB_MEMORY_LIMIT",        "1GB"),
+    "duckdb_io_multiplier":       ("SUPERTABLE_DUCKDB_IO_MULTIPLIER",       "3"),
+    "duckdb_threads":             ("SUPERTABLE_DUCKDB_THREADS",             ""),
+    "duckdb_http_timeout":        ("SUPERTABLE_DUCKDB_HTTP_TIMEOUT",        ""),
+    "duckdb_external_cache_size": ("SUPERTABLE_DUCKDB_EXTERNAL_CACHE_SIZE", "5GB"),
+}
+
+# The two DuckDB engines that each carry their own runtime section.
+DUCKDB_ENGINES: Tuple[str, ...] = ("lite", "pro")
+
+# Public whitelists (for callers / catalog / tests).
+SHARED_CONFIG_FIELDS: Tuple[str, ...] = tuple(_SHARED_SPEC.keys())
+DUCKDB_CONFIG_FIELDS: Tuple[str, ...] = tuple(_DUCKDB_SPEC.keys())
+ENGINE_CONFIG_FIELDS: Tuple[str, ...] = SHARED_CONFIG_FIELDS + DUCKDB_CONFIG_FIELDS
+
+
+@dataclass(frozen=True)
+class EngineRuntimeConfig:
+    """Effective, typed config for one DuckDB engine resolution.
+
+    Carries the shared auto-pick thresholds (identical across engines) plus the
+    DuckDB session pragmas for the engine it was resolved for.
+    """
+
+    # auto-pick thresholds (shared)
+    engine_lite_max_bytes: int
+    engine_spark_min_bytes: int
+    engine_freshness_sec: int
+
+    # duckdb session pragmas (per-engine, re-applied live per query)
+    duckdb_memory_limit: str             # e.g. "1GB"
+    duckdb_io_multiplier: float
+    duckdb_threads: Optional[int]        # None → auto-derive from memory limit
+    duckdb_http_timeout: Optional[int]   # seconds; None → leave DuckDB default
+    duckdb_external_cache_size: str      # "" → external file cache disabled
+
+
+def _redis_cfg(org: str, catalog: Optional[Any]) -> Dict[str, Any]:
+    """Read the stored engine config for ``org`` (best effort).
+
+    Returns an empty dict when no catalog is supplied or Redis is unreachable,
+    so the resolver degrades cleanly to env vars + defaults.
+    """
+    if not (org and catalog is not None):
+        return {}
+    try:
+        return catalog.get_engine_config(org) or {}
+    except Exception:
+        return {}
+
+
+def _effective(stored: Dict[str, Any], key: str, spec: Dict[str, Tuple[str, str]]) -> Tuple[str, str]:
+    """Return ``(value, source)`` for one field. source ∈ redis|env|default.
+
+    ``stored`` is the dict the value is looked up in directly: the top-level
+    document for shared fields, or the ``lite`` / ``pro`` sub-section for
+    per-engine fields.
+    """
+    env_var, default = spec[key]
+    redis_val = stored.get(key) if isinstance(stored, dict) else None
+    if redis_val is not None and str(redis_val).strip() != "":
+        return str(redis_val).strip(), "redis"
+    env_val = (os.getenv(env_var) or "").strip()
+    if env_val:
+        return env_val, "env"
+    return default, "default"
+
+
+def _to_int(s: Any, fallback: int) -> int:
+    try:
+        return int(str(s).strip())
+    except (TypeError, ValueError):
+        return fallback
+
+
+def _to_float(s: Any, fallback: float) -> float:
+    try:
+        return float(str(s).strip())
+    except (TypeError, ValueError):
+        return fallback
+
+
+def _build_runtime(redis_cfg: Dict[str, Any], engine: str) -> EngineRuntimeConfig:
+    """Build the typed config for ``engine`` ("lite"/"pro") from a stored doc."""
+    section = redis_cfg.get(engine) if isinstance(redis_cfg.get(engine), dict) else {}
+
+    sv = {k: _effective(redis_cfg, k, _SHARED_SPEC)[0] for k in _SHARED_SPEC}
+    dv = {k: _effective(section, k, _DUCKDB_SPEC)[0] for k in _DUCKDB_SPEC}
+
+    threads = _to_int(dv["duckdb_threads"], 0) if dv["duckdb_threads"] else 0
+    http = _to_int(dv["duckdb_http_timeout"], 0) if dv["duckdb_http_timeout"] else 0
+
+    return EngineRuntimeConfig(
+        engine_lite_max_bytes=_to_int(sv["engine_lite_max_bytes"], 100 * 1024 * 1024),
+        engine_spark_min_bytes=_to_int(sv["engine_spark_min_bytes"], 10 * 1024 * 1024 * 1024),
+        engine_freshness_sec=_to_int(sv["engine_freshness_sec"], 300),
+        duckdb_memory_limit=normalize_memory_size(dv["duckdb_memory_limit"], default="1GB"),
+        duckdb_io_multiplier=_to_float(dv["duckdb_io_multiplier"], 3.0),
+        duckdb_threads=(threads if threads > 0 else None),
+        duckdb_http_timeout=(http if http > 0 else None),
+        duckdb_external_cache_size=normalize_memory_size(dv["duckdb_external_cache_size"], default=""),
+    )
+
+
+def resolve_engine_configs(org: str, catalog: Optional[Any] = None) -> Dict[str, EngineRuntimeConfig]:
+    """Resolve both DuckDB engines in a single Redis read.
+
+    Returns ``{"lite": EngineRuntimeConfig, "pro": EngineRuntimeConfig}``.  The
+    shared thresholds are identical in both; only the DuckDB pragmas differ.
+    """
+    cfg = _redis_cfg(org, catalog)
+    return {e: _build_runtime(cfg, e) for e in DUCKDB_ENGINES}
+
+
+def resolve_engine_config(org: str, catalog: Optional[Any] = None, engine: str = "lite") -> EngineRuntimeConfig:
+    """Resolve effective config for a single ``engine`` (Redis → env → default).
+
+    ``catalog`` is an optional ``RedisCatalog``.  When it is None or Redis is
+    unreachable the resolver falls back to environment variables and built-in
+    defaults, which keeps the engine usable without Redis (e.g. in unit tests).
+    """
+    engine = engine if engine in DUCKDB_ENGINES else "lite"
+    return _build_runtime(_redis_cfg(org, catalog), engine)
+
+
+def resolve_engine_config_provenance(org: str, catalog: Optional[Any] = None) -> Dict[str, Any]:
+    """Per-field provenance for the UI.
+
+    Shape::
+
+        {
+          "shared": {field: {value, source, env_var, default}, ...},
+          "lite":   {field: {value, source, env_var, default}, ...},
+          "pro":    {field: {value, source, env_var, default}, ...},
+          "modified_ms": <int|None>,
+        }
+
+    ``source`` is one of ``redis`` / ``env`` / ``default``.
+    """
+    cfg = _redis_cfg(org, catalog)
+
+    def _block(stored: Dict[str, Any], spec: Dict[str, Tuple[str, str]]) -> Dict[str, Any]:
+        out: Dict[str, Any] = {}
+        for key, (env_var, default) in spec.items():
+            value, source = _effective(stored, key, spec)
+            out[key] = {"value": value, "source": source, "env_var": env_var, "default": default}
+        return out
+
+    result: Dict[str, Any] = {"shared": _block(cfg, _SHARED_SPEC)}
+    for engine in DUCKDB_ENGINES:
+        section = cfg.get(engine) if isinstance(cfg.get(engine), dict) else {}
+        result[engine] = _block(section, _DUCKDB_SPEC)
+    result["modified_ms"] = cfg.get("modified_ms")
+    return result

{supertable-2.2.5 → supertable-2.2.7}/supertable/engine/executor.py

@@ -16,7 +16,7 @@ from supertable.utils.sql_parser import SQLParser
 from supertable.engine.engine_enum import Engine
 from supertable.engine.duckdb_lite import DuckDBLite
 from supertable.engine.duckdb_pro import DuckDBPro
-from supertable.engine.engine_config import resolve_engine_config, EngineRuntimeConfig
+from supertable.engine.engine_config import resolve_engine_configs, EngineRuntimeConfig
 from supertable.data_classes import Reflection
 from supertable.config.defaults import logger
 
@@ -62,6 +62,47 @@ class Executor:
                 self._catalog = False  # sentinel: construction failed, do not retry
         return self._catalog or None
 
+    def _spark_min_bytes(self, cfg: EngineRuntimeConfig) -> int:
+        """Effective lower byte bound for routing a query to Spark under AUTO.
+
+        Couples the configured policy floor (``engine_spark_min_bytes``) with the
+        registered Spark fleet so the boundary is::
+
+            max(engine_spark_min_bytes, min(active cluster min_bytes))
+
+        Rationale:
+          * The ``max`` keeps the admin's global floor effective — keep
+            ``engine_spark_min_bytes`` high to hold medium jobs on DuckDB even
+            when a cluster would accept them (set it to 0 to let the fleet alone
+            drive routing).
+          * Folding in the fleet minimum means AUTO never routes a job *below*
+            what any active cluster accepts — which would otherwise make
+            ``select_spark_cluster`` return nothing and hard-fail the query
+            (see spark_thrift.py ``_select_cluster``).
+
+        Degrades to the policy floor alone when no catalog is reachable or no
+        active clusters are registered.
+        """
+        policy = cfg.engine_spark_min_bytes
+        catalog = self._get_catalog()
+        if catalog is None:
+            return policy
+        try:
+            clusters = catalog.list_spark_clusters(self.organization) or []
+        except Exception:
+            return policy
+        mins = []
+        for c in clusters:
+            if not isinstance(c, dict) or c.get("status") != "active":
+                continue
+            try:
+                mins.append(int(c.get("min_bytes", 0)))
+            except (TypeError, ValueError):
+                continue
+        if not mins:
+            return policy
+        return max(policy, min(mins))
+
     def _auto_pick(self, reflection: Reflection, cfg: EngineRuntimeConfig) -> Engine:
         """Select the best engine based on data size and freshness.
 
@@ -86,12 +127,17 @@ class Executor:
           SUPERTABLE_ENGINE_LITE_MAX_BYTES   – upper bound for Lite (default 100 MB)
           SUPERTABLE_ENGINE_SPARK_MIN_BYTES  – lower bound for Spark (default 10 GB)
           SUPERTABLE_ENGINE_FRESHNESS_SEC    – age threshold in seconds (default 300)
+
+        The Spark lower bound is additionally coupled to the registered fleet
+        (see :meth:`_spark_min_bytes`): the effective boundary is the larger of
+        the configured floor and the smallest ``min_bytes`` across active
+        clusters, so AUTO only routes to Spark when some cluster will take it.
         """
         bytes_total = reflection.reflection_bytes
 
         # --- thresholds (resolved live from org system config) ---
         lite_max = cfg.engine_lite_max_bytes
-        spark_min = cfg.engine_spark_min_bytes
+        spark_min = self._spark_min_bytes(cfg)
         freshness_threshold_s = cfg.engine_freshness_sec
 
         # --- freshness: how long ago was the most recent snapshot updated ---
@@ -143,10 +189,14 @@ class Executor:
         log_prefix: str,
     ) -> Tuple[pd.DataFrame, str]:
         # Resolve engine config live (Redis → env → default) for this query so
-        # UI changes take effect immediately without restart or cache.
-        cfg = resolve_engine_config(self.organization, self._get_catalog())
+        # UI changes take effect immediately without restart or cache.  Lite and
+        # Pro carry independent DuckDB pragmas; the shared auto-pick thresholds
+        # are identical in both, so either may drive the routing decision.
+        cfgs = resolve_engine_configs(self.organization, self._get_catalog())
+        lite_cfg = cfgs["lite"]
+        pro_cfg = cfgs["pro"]
 
-        chosen = engine if engine != Engine.AUTO else self._auto_pick(reflection, cfg)
+        chosen = engine if engine != Engine.AUTO else self._auto_pick(reflection, lite_cfg)
 
         def timer_capture(evt: str):
             timer.capture_and_reset_timing(evt)
@@ -158,7 +208,7 @@ class Executor:
                 query_manager=query_manager,
                 timer_capture=timer_capture,
                 log_prefix=log_prefix,
-                engine_config=cfg,
+                engine_config=lite_cfg,
             )
             used = "duckdb_lite"
 
@@ -170,7 +220,7 @@ class Executor:
                 query_manager=query_manager,
                 timer_capture=timer_capture,
                 log_prefix=log_prefix,
-                engine_config=cfg,
+                engine_config=pro_cfg,
             )
             used = "duckdb_pro"
 

{supertable-2.2.5 → supertable-2.2.7}/supertable/engine/spark_thrift.py

@@ -25,7 +25,7 @@ from supertable.engine.engine_common import (
     escape_parquet_path,
 )
 
-from urllib.parse import urlparse
+from urllib.parse import urlparse, unquote
 
 # Suppress verbose INFO logging from PyHive and Thrift libraries.
 # PyHive logs every SQL statement (CREATE VIEW, DROP VIEW, SET, etc.)
@@ -77,8 +77,14 @@ def _to_s3a_path(file_path: str) -> str:
 
     if file_path.startswith("http://") or file_path.startswith("https://"):
         parsed = urlparse(file_path)
-        # path is /bucket/key — strip the leading slash, split into bucket + key
-        path = parsed.path.lstrip("/")
+        # path is /bucket/key — strip the leading slash, split into bucket + key.
+        # urlparse leaves percent-escapes intact, and boto3 presigning encodes
+        # the key (Hive partition dirs like ``year=2006`` become ``year%3D2006``).
+        # Spark's S3A client expects the *raw* object key and re-encodes itself,
+        # so we must decode here — otherwise Ceph RGW, which matches keys
+        # literally, 404s on the ``%3D`` form. The presign query string is
+        # dropped on purpose: Spark authenticates with its own fs.s3a.* creds.
+        path = unquote(parsed.path).lstrip("/")
         if "/" in path:
             return f"s3a://{path}"
         # Degenerate case: only bucket, no key — return as-is