supertable 2.2.4__tar.gz → 2.2.6__tar.gz

{supertable-2.2.4/supertable.egg-info → supertable-2.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supertable
-Version: 2.2.4
+Version: 2.2.6
 Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
 Author: Levente Kupas
 Author-email: Levente Kupas <lkupas@kladnasoft.com>

{supertable-2.2.4 → supertable-2.2.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "supertable"
-version = "2.2.4"
+version = "2.2.6"
 description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
 readme = "README.md"
 requires-python = ">=3.10"

{supertable-2.2.4 → supertable-2.2.6}/setup.py

@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
 
 setup(
     name="supertable",
-    version="2.2.4",
+    version="2.2.6",
     description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
     long_description=long_description,
     long_description_content_type="text/markdown",

{supertable-2.2.4 → supertable-2.2.6}/supertable/__init__.py

@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
 project documentation for the full API surface.
 """
 
-__version__ = "2.2.4"
+__version__ = "2.2.6"
 
 # Re-export the core public surface so users can do ``from supertable import …``
 # instead of remembering submodule paths.

{supertable-2.2.4 → supertable-2.2.6}/supertable/engine/duckdb_lite.py

@@ -21,6 +21,7 @@ from supertable.engine.engine_common import (
     create_reflection_view_with_presign_retry,
     rewrite_query_with_hashed_tables,
     init_connection,
+    apply_runtime_pragmas,
     create_rbac_view,
     create_dedup_view,
     create_tombstone_view,
@@ -105,6 +106,7 @@ class DuckDBLite:
             query_manager: QueryPlanManager,
             timer_capture,
             log_prefix: str = "",
+            engine_config=None,
     ) -> pd.DataFrame:
         tried_presign = False
 
@@ -237,6 +239,10 @@ class DuckDBLite:
             except Exception:
                 pass
 
+            # Re-apply live engine config (memory/threads/http/cache) so UI
+            # changes take effect on this persistent connection per query.
+            apply_runtime_pragmas(con, engine_config)
+
             logger.debug(f"{log_prefix}[duckdb.lite] executing: {executing_query}")
             result = con.execute(executing_query).fetchdf()
 

{supertable-2.2.4 → supertable-2.2.6}/supertable/engine/duckdb_pro.py

@@ -25,6 +25,7 @@ from supertable.engine.engine_common import (
     make_presigned_list,
     rewrite_query_with_hashed_tables,
     init_connection,
+    apply_runtime_pragmas,
     create_rbac_view,
     create_dedup_view,
     create_tombstone_view,
@@ -268,6 +269,7 @@ class DuckDBPro:
             query_manager: QueryPlanManager,
             timer_capture,
             log_prefix: str = "",
+            engine_config=None,
     ) -> pd.DataFrame:
         tables_used: Set[str] = set()
 
@@ -368,6 +370,10 @@ class DuckDBPro:
 
             logger.debug(f"{log_prefix}[duckdb.pro] executing: {executing_query}")
 
+            # Re-apply live engine config (memory/threads/http/cache) so UI
+            # changes take effect on this persistent connection per query.
+            apply_runtime_pragmas(con, engine_config)
+
             # Profiling PRAGMAs are connection-level state.  Under concurrent
             # queries the last SET wins — one query's profile may land in the
             # wrong file.  This is acceptable: profiling is best-effort

{supertable-2.2.4 → supertable-2.2.6}/supertable/engine/engine_common.py

@@ -140,7 +140,11 @@ def _parse_memory_limit_mb(value: str) -> Optional[int]:
     return int(amount)
 
 
-def _derive_thread_count(memory_limit_str: str, fallback: int = 2) -> int:
+def _derive_thread_count(
+        memory_limit_str: str,
+        fallback: int = 2,
+        multiplier: Optional[float] = None,
+) -> int:
     """Derive a safe DuckDB thread count for remote parquet reads.
 
     DuckDB uses synchronous IO: each thread can make at most one HTTP/S3
@@ -170,9 +174,10 @@ def _derive_thread_count(memory_limit_str: str, fallback: int = 2) -> int:
         return fallback
 
     cpu_count = _os.cpu_count() or fallback
-    multiplier = settings.SUPERTABLE_DUCKDB_IO_MULTIPLIER
+    if multiplier is None:
+        multiplier = settings.SUPERTABLE_DUCKDB_IO_MULTIPLIER
 
-    io_threads = cpu_count * multiplier
+    io_threads = int(cpu_count * multiplier)
     # Safety ceiling: never allocate fewer than ~400 MB per thread.
     memory_floor = max(1, mb // 400)
     result = min(io_threads, memory_floor)
@@ -625,6 +630,56 @@ def init_connection(
         pass
 
 
+def apply_runtime_pragmas(con: duckdb.DuckDBPyConnection, cfg) -> None:
+    """Re-apply the session-settable DuckDB pragmas from a live engine config.
+
+    DuckDB Lite/Pro reuse a persistent connection, so settings applied once at
+    ``init_connection`` time would otherwise freeze for the connection's life.
+    Calling this immediately before each query makes the org's engine config
+    (memory limit, thread count, HTTP timeout, external file cache) take effect
+    live — the connection adopts the latest values without being torn down.
+
+    ``cfg`` is an ``EngineRuntimeConfig``.  When None the connection keeps
+    whatever ``init_connection`` applied (settings-based), so callers without a
+    resolved config are unaffected.  Every pragma is best-effort: httpfs-only
+    settings silently no-op on connections that have not loaded httpfs.
+    """
+    if cfg is None:
+        return
+
+    memory_limit = (cfg.duckdb_memory_limit or "").strip() or "1GB"
+    try:
+        con.execute(f"PRAGMA memory_limit='{memory_limit}';")
+    except Exception:
+        pass
+
+    # Explicit thread count wins; otherwise derive from the live memory limit
+    # and IO multiplier (same formula as init_connection).
+    if cfg.duckdb_threads is not None:
+        thread_count = cfg.duckdb_threads
+    else:
+        thread_count = _derive_thread_count(memory_limit, multiplier=cfg.duckdb_io_multiplier)
+    try:
+        con.execute(f"SET threads={thread_count};")
+    except Exception:
+        pass
+
+    # httpfs settings — only effective once httpfs is loaded (remote reads).
+    if cfg.duckdb_http_timeout is not None:
+        try:
+            con.execute(f"SET http_timeout={int(cfg.duckdb_http_timeout)};")
+        except Exception:
+            pass
+
+    cache_size = (cfg.duckdb_external_cache_size or "").strip()
+    if cache_size:
+        try:
+            con.execute("SET enable_external_file_cache=true;")
+            con.execute(f"SET external_file_cache_max_size='{sanitize_sql_string(cache_size)}';")
+        except Exception:
+            pass
+
+
 # =========================================================
 # RBAC view creation
 # =========================================================

supertable-2.2.6/supertable/engine/engine_config.py

@@ -0,0 +1,228 @@
+# route: supertable.engine.engine_config
+"""
+Engine runtime configuration — single source of truth for resolution.
+
+Engine config is stored once per organization at **system scope**
+(``supertable:{org}:system:engine:duckdb``).  It is global to the org, not
+per-supertable, and is resolved **live on every query** (no caching) so that
+changes made in the UI take effect on the next query without a restart or a
+connection reset.
+
+Two kinds of settings live in that document:
+
+  * **Shared auto-pick thresholds** (one value, used by the router):
+    ``engine_lite_max_bytes``, ``engine_spark_min_bytes``,
+    ``engine_freshness_sec``.  These describe the *boundaries* between engines
+    so they cannot be per-engine.
+  * **Per-engine DuckDB runtime pragmas** (separate values for Lite and Pro):
+    ``duckdb_memory_limit``, ``duckdb_io_multiplier``, ``duckdb_threads``,
+    ``duckdb_http_timeout``, ``duckdb_external_cache_size``.  Each DuckDB engine
+    carries its own copy under a ``"lite"`` / ``"pro"`` section, so tuning Lite
+    never changes Pro and vice-versa.
+
+Stored document shape::
+
+    {
+      "engine_lite_max_bytes": "104857600",
+      "engine_spark_min_bytes": "10737418240",
+      "engine_freshness_sec":   "300",
+      "lite": {"duckdb_memory_limit": "1GB", ...},
+      "pro":  {"duckdb_memory_limit": "8GB", ...},
+      "modified_ms": 1750000000000
+    }
+
+Resolution precedence, per field::
+
+    Redis (org system config)  →  environment variable  →  built-in default
+
+The environment variable is a single global fallback shared by both engines;
+per-engine differentiation therefore only happens once a value is stored in
+Redis (via the UI).
+
+Entry points (all share that precedence so engine and UI never disagree):
+
+  * :func:`resolve_engine_configs`           → ``{"lite": cfg, "pro": cfg}`` in a
+                                                single Redis read; used by the
+                                                executor.
+  * :func:`resolve_engine_config`            → typed :class:`EngineRuntimeConfig`
+                                                for one engine (convenience).
+  * :func:`resolve_engine_config_provenance` → ``{shared, lite, pro,
+                                                modified_ms}`` per-field
+                                                ``{value, source, env_var,
+                                                default}`` for the API / UI.
+
+Built-in defaults mirror the :mod:`supertable.config.settings` dataclass
+defaults, so behaviour is identical whether a value flows through ``settings``
+(no Redis override) or through this resolver.
+"""
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Any, Dict, Optional, Tuple
+
+
+# Shared auto-pick thresholds: key → (env var, built-in default as string).
+_SHARED_SPEC: Dict[str, Tuple[str, str]] = {
+    "engine_lite_max_bytes":  ("SUPERTABLE_ENGINE_LITE_MAX_BYTES",  str(100 * 1024 * 1024)),
+    "engine_spark_min_bytes": ("SUPERTABLE_ENGINE_SPARK_MIN_BYTES", str(10 * 1024 * 1024 * 1024)),
+    "engine_freshness_sec":   ("SUPERTABLE_ENGINE_FRESHNESS_SEC",   "300"),
+}
+
+# Per-engine DuckDB runtime pragmas: key → (env var, built-in default as string).
+# Defaults mirror the settings dataclass defaults in supertable/config/settings.py.
+_DUCKDB_SPEC: Dict[str, Tuple[str, str]] = {
+    "duckdb_memory_limit":        ("SUPERTABLE_DUCKDB_MEMORY_LIMIT",        "1GB"),
+    "duckdb_io_multiplier":       ("SUPERTABLE_DUCKDB_IO_MULTIPLIER",       "3"),
+    "duckdb_threads":             ("SUPERTABLE_DUCKDB_THREADS",             ""),
+    "duckdb_http_timeout":        ("SUPERTABLE_DUCKDB_HTTP_TIMEOUT",        ""),
+    "duckdb_external_cache_size": ("SUPERTABLE_DUCKDB_EXTERNAL_CACHE_SIZE", "5GB"),
+}
+
+# The two DuckDB engines that each carry their own runtime section.
+DUCKDB_ENGINES: Tuple[str, ...] = ("lite", "pro")
+
+# Public whitelists (for callers / catalog / tests).
+SHARED_CONFIG_FIELDS: Tuple[str, ...] = tuple(_SHARED_SPEC.keys())
+DUCKDB_CONFIG_FIELDS: Tuple[str, ...] = tuple(_DUCKDB_SPEC.keys())
+ENGINE_CONFIG_FIELDS: Tuple[str, ...] = SHARED_CONFIG_FIELDS + DUCKDB_CONFIG_FIELDS
+
+
+@dataclass(frozen=True)
+class EngineRuntimeConfig:
+    """Effective, typed config for one DuckDB engine resolution.
+
+    Carries the shared auto-pick thresholds (identical across engines) plus the
+    DuckDB session pragmas for the engine it was resolved for.
+    """
+
+    # auto-pick thresholds (shared)
+    engine_lite_max_bytes: int
+    engine_spark_min_bytes: int
+    engine_freshness_sec: int
+
+    # duckdb session pragmas (per-engine, re-applied live per query)
+    duckdb_memory_limit: str             # e.g. "1GB"
+    duckdb_io_multiplier: float
+    duckdb_threads: Optional[int]        # None → auto-derive from memory limit
+    duckdb_http_timeout: Optional[int]   # seconds; None → leave DuckDB default
+    duckdb_external_cache_size: str      # "" → external file cache disabled
+
+
+def _redis_cfg(org: str, catalog: Optional[Any]) -> Dict[str, Any]:
+    """Read the stored engine config for ``org`` (best effort).
+
+    Returns an empty dict when no catalog is supplied or Redis is unreachable,
+    so the resolver degrades cleanly to env vars + defaults.
+    """
+    if not (org and catalog is not None):
+        return {}
+    try:
+        return catalog.get_engine_config(org) or {}
+    except Exception:
+        return {}
+
+
+def _effective(stored: Dict[str, Any], key: str, spec: Dict[str, Tuple[str, str]]) -> Tuple[str, str]:
+    """Return ``(value, source)`` for one field. source ∈ redis|env|default.
+
+    ``stored`` is the dict the value is looked up in directly: the top-level
+    document for shared fields, or the ``lite`` / ``pro`` sub-section for
+    per-engine fields.
+    """
+    env_var, default = spec[key]
+    redis_val = stored.get(key) if isinstance(stored, dict) else None
+    if redis_val is not None and str(redis_val).strip() != "":
+        return str(redis_val).strip(), "redis"
+    env_val = (os.getenv(env_var) or "").strip()
+    if env_val:
+        return env_val, "env"
+    return default, "default"
+
+
+def _to_int(s: Any, fallback: int) -> int:
+    try:
+        return int(str(s).strip())
+    except (TypeError, ValueError):
+        return fallback
+
+
+def _to_float(s: Any, fallback: float) -> float:
+    try:
+        return float(str(s).strip())
+    except (TypeError, ValueError):
+        return fallback
+
+
+def _build_runtime(redis_cfg: Dict[str, Any], engine: str) -> EngineRuntimeConfig:
+    """Build the typed config for ``engine`` ("lite"/"pro") from a stored doc."""
+    section = redis_cfg.get(engine) if isinstance(redis_cfg.get(engine), dict) else {}
+
+    sv = {k: _effective(redis_cfg, k, _SHARED_SPEC)[0] for k in _SHARED_SPEC}
+    dv = {k: _effective(section, k, _DUCKDB_SPEC)[0] for k in _DUCKDB_SPEC}
+
+    threads = _to_int(dv["duckdb_threads"], 0) if dv["duckdb_threads"] else 0
+    http = _to_int(dv["duckdb_http_timeout"], 0) if dv["duckdb_http_timeout"] else 0
+
+    return EngineRuntimeConfig(
+        engine_lite_max_bytes=_to_int(sv["engine_lite_max_bytes"], 100 * 1024 * 1024),
+        engine_spark_min_bytes=_to_int(sv["engine_spark_min_bytes"], 10 * 1024 * 1024 * 1024),
+        engine_freshness_sec=_to_int(sv["engine_freshness_sec"], 300),
+        duckdb_memory_limit=dv["duckdb_memory_limit"].strip() or "1GB",
+        duckdb_io_multiplier=_to_float(dv["duckdb_io_multiplier"], 3.0),
+        duckdb_threads=(threads if threads > 0 else None),
+        duckdb_http_timeout=(http if http > 0 else None),
+        duckdb_external_cache_size=dv["duckdb_external_cache_size"].strip(),
+    )
+
+
+def resolve_engine_configs(org: str, catalog: Optional[Any] = None) -> Dict[str, EngineRuntimeConfig]:
+    """Resolve both DuckDB engines in a single Redis read.
+
+    Returns ``{"lite": EngineRuntimeConfig, "pro": EngineRuntimeConfig}``.  The
+    shared thresholds are identical in both; only the DuckDB pragmas differ.
+    """
+    cfg = _redis_cfg(org, catalog)
+    return {e: _build_runtime(cfg, e) for e in DUCKDB_ENGINES}
+
+
+def resolve_engine_config(org: str, catalog: Optional[Any] = None, engine: str = "lite") -> EngineRuntimeConfig:
+    """Resolve effective config for a single ``engine`` (Redis → env → default).
+
+    ``catalog`` is an optional ``RedisCatalog``.  When it is None or Redis is
+    unreachable the resolver falls back to environment variables and built-in
+    defaults, which keeps the engine usable without Redis (e.g. in unit tests).
+    """
+    engine = engine if engine in DUCKDB_ENGINES else "lite"
+    return _build_runtime(_redis_cfg(org, catalog), engine)
+
+
+def resolve_engine_config_provenance(org: str, catalog: Optional[Any] = None) -> Dict[str, Any]:
+    """Per-field provenance for the UI.
+
+    Shape::
+
+        {
+          "shared": {field: {value, source, env_var, default}, ...},
+          "lite":   {field: {value, source, env_var, default}, ...},
+          "pro":    {field: {value, source, env_var, default}, ...},
+          "modified_ms": <int|None>,
+        }
+
+    ``source`` is one of ``redis`` / ``env`` / ``default``.
+    """
+    cfg = _redis_cfg(org, catalog)
+
+    def _block(stored: Dict[str, Any], spec: Dict[str, Tuple[str, str]]) -> Dict[str, Any]:
+        out: Dict[str, Any] = {}
+        for key, (env_var, default) in spec.items():
+            value, source = _effective(stored, key, spec)
+            out[key] = {"value": value, "source": source, "env_var": env_var, "default": default}
+        return out
+
+    result: Dict[str, Any] = {"shared": _block(cfg, _SHARED_SPEC)}
+    for engine in DUCKDB_ENGINES:
+        section = cfg.get(engine) if isinstance(cfg.get(engine), dict) else {}
+        result[engine] = _block(section, _DUCKDB_SPEC)
+    result["modified_ms"] = cfg.get("modified_ms")
+    return result

{supertable-2.2.4 → supertable-2.2.6}/supertable/engine/executor.py

@@ -16,9 +16,9 @@ from supertable.utils.sql_parser import SQLParser
 from supertable.engine.engine_enum import Engine
 from supertable.engine.duckdb_lite import DuckDBLite
 from supertable.engine.duckdb_pro import DuckDBPro
+from supertable.engine.engine_config import resolve_engine_configs, EngineRuntimeConfig
 from supertable.data_classes import Reflection
 from supertable.config.defaults import logger
-from supertable.config.settings import settings
 
 
 # Module-level singleton for the pro executor so the persistent
@@ -46,8 +46,23 @@ class Executor:
         self.organization = organization
         self.lite_exec = DuckDBLite(storage=storage)
         self.spark_exec = None
+        self._catalog = None  # lazily created RedisCatalog for live config reads
 
-    def _auto_pick(self, reflection: Reflection) -> Engine:
+    def _get_catalog(self):
+        """Lazily create a RedisCatalog for live engine-config reads.
+
+        Returns None when Redis is unreachable so config resolution degrades to
+        environment variables and built-in defaults instead of failing a query.
+        """
+        if self._catalog is None:
+            try:
+                from supertable.redis_catalog import RedisCatalog
+                self._catalog = RedisCatalog()
+            except Exception:
+                self._catalog = False  # sentinel: construction failed, do not retry
+        return self._catalog or None
+
+    def _auto_pick(self, reflection: Reflection, cfg: EngineRuntimeConfig) -> Engine:
         """Select the best engine based on data size and freshness.
 
         Decision matrix (all thresholds configurable via env vars):
@@ -74,21 +89,10 @@ class Executor:
         """
         bytes_total = reflection.reflection_bytes
 
-        # --- thresholds (env-configurable) ---
-        try:
-            lite_max = settings.SUPERTABLE_ENGINE_LITE_MAX_BYTES
-        except ValueError:
-            lite_max = 100 * 1024 * 1024  # 100 MB
-
-        try:
-            spark_min = settings.SUPERTABLE_ENGINE_SPARK_MIN_BYTES
-        except ValueError:
-            spark_min = 10 * 1024 * 1024 * 1024  # 10 GB
-
-        try:
-            freshness_threshold_s = settings.SUPERTABLE_ENGINE_FRESHNESS_SEC
-        except ValueError:
-            freshness_threshold_s = 300  # 5 min
+        # --- thresholds (resolved live from org system config) ---
+        lite_max = cfg.engine_lite_max_bytes
+        spark_min = cfg.engine_spark_min_bytes
+        freshness_threshold_s = cfg.engine_freshness_sec
 
         # --- freshness: how long ago was the most recent snapshot updated ---
         if reflection.freshness_ms > 0:
@@ -138,7 +142,15 @@ class Executor:
         plan_stats: PlanStats,
         log_prefix: str,
     ) -> Tuple[pd.DataFrame, str]:
-        chosen = engine if engine != Engine.AUTO else self._auto_pick(reflection)
+        # Resolve engine config live (Redis → env → default) for this query so
+        # UI changes take effect immediately without restart or cache.  Lite and
+        # Pro carry independent DuckDB pragmas; the shared auto-pick thresholds
+        # are identical in both, so either may drive the routing decision.
+        cfgs = resolve_engine_configs(self.organization, self._get_catalog())
+        lite_cfg = cfgs["lite"]
+        pro_cfg = cfgs["pro"]
+
+        chosen = engine if engine != Engine.AUTO else self._auto_pick(reflection, lite_cfg)
 
         def timer_capture(evt: str):
             timer.capture_and_reset_timing(evt)
@@ -150,6 +162,7 @@ class Executor:
                 query_manager=query_manager,
                 timer_capture=timer_capture,
                 log_prefix=log_prefix,
+                engine_config=lite_cfg,
             )
             used = "duckdb_lite"
 
@@ -161,6 +174,7 @@ class Executor:
                 query_manager=query_manager,
                 timer_capture=timer_capture,
                 log_prefix=log_prefix,
+                engine_config=pro_cfg,
             )
             used = "duckdb_pro"
 

{supertable-2.2.4 → supertable-2.2.6}/supertable/engine/spark_thrift.py

@@ -25,7 +25,7 @@ from supertable.engine.engine_common import (
     escape_parquet_path,
 )
 
-from urllib.parse import urlparse
+from urllib.parse import urlparse, unquote
 
 # Suppress verbose INFO logging from PyHive and Thrift libraries.
 # PyHive logs every SQL statement (CREATE VIEW, DROP VIEW, SET, etc.)
@@ -77,8 +77,14 @@ def _to_s3a_path(file_path: str) -> str:
 
     if file_path.startswith("http://") or file_path.startswith("https://"):
         parsed = urlparse(file_path)
-        # path is /bucket/key — strip the leading slash, split into bucket + key
-        path = parsed.path.lstrip("/")
+        # path is /bucket/key — strip the leading slash, split into bucket + key.
+        # urlparse leaves percent-escapes intact, and boto3 presigning encodes
+        # the key (Hive partition dirs like ``year=2006`` become ``year%3D2006``).
+        # Spark's S3A client expects the *raw* object key and re-encodes itself,
+        # so we must decode here — otherwise Ceph RGW, which matches keys
+        # literally, 404s on the ``%3D`` form. The presign query string is
+        # dropped on purpose: Spark authenticates with its own fs.s3a.* creds.
+        path = unquote(parsed.path).lstrip("/")
         if "/" in path:
             return f"s3a://{path}"
         # Degenerate case: only bucket, no key — return as-is