supertable 2.2.4__tar.gz → 2.2.5__tar.gz

{supertable-2.2.4/supertable.egg-info → supertable-2.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supertable
-Version: 2.2.4
+Version: 2.2.5
 Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
 Author: Levente Kupas
 Author-email: Levente Kupas <lkupas@kladnasoft.com>

{supertable-2.2.4 → supertable-2.2.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "supertable"
-version = "2.2.4"
+version = "2.2.5"
 description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
 readme = "README.md"
 requires-python = ">=3.10"

{supertable-2.2.4 → supertable-2.2.5}/setup.py

@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
 
 setup(
     name="supertable",
-    version="2.2.4",
+    version="2.2.5",
     description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
     long_description=long_description,
     long_description_content_type="text/markdown",

{supertable-2.2.4 → supertable-2.2.5}/supertable/__init__.py

@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
 project documentation for the full API surface.
 """
 
-__version__ = "2.2.4"
+__version__ = "2.2.5"
 
 # Re-export the core public surface so users can do ``from supertable import …``
 # instead of remembering submodule paths.

{supertable-2.2.4 → supertable-2.2.5}/supertable/engine/duckdb_lite.py

@@ -21,6 +21,7 @@ from supertable.engine.engine_common import (
     create_reflection_view_with_presign_retry,
     rewrite_query_with_hashed_tables,
     init_connection,
+    apply_runtime_pragmas,
     create_rbac_view,
     create_dedup_view,
     create_tombstone_view,
@@ -105,6 +106,7 @@ class DuckDBLite:
             query_manager: QueryPlanManager,
             timer_capture,
             log_prefix: str = "",
+            engine_config=None,
     ) -> pd.DataFrame:
         tried_presign = False
 
@@ -237,6 +239,10 @@ class DuckDBLite:
             except Exception:
                 pass
 
+            # Re-apply live engine config (memory/threads/http/cache) so UI
+            # changes take effect on this persistent connection per query.
+            apply_runtime_pragmas(con, engine_config)
+
             logger.debug(f"{log_prefix}[duckdb.lite] executing: {executing_query}")
             result = con.execute(executing_query).fetchdf()
 

{supertable-2.2.4 → supertable-2.2.5}/supertable/engine/duckdb_pro.py

@@ -25,6 +25,7 @@ from supertable.engine.engine_common import (
     make_presigned_list,
     rewrite_query_with_hashed_tables,
     init_connection,
+    apply_runtime_pragmas,
     create_rbac_view,
     create_dedup_view,
     create_tombstone_view,
@@ -268,6 +269,7 @@ class DuckDBPro:
             query_manager: QueryPlanManager,
             timer_capture,
             log_prefix: str = "",
+            engine_config=None,
     ) -> pd.DataFrame:
         tables_used: Set[str] = set()
 
@@ -368,6 +370,10 @@ class DuckDBPro:
 
             logger.debug(f"{log_prefix}[duckdb.pro] executing: {executing_query}")
 
+            # Re-apply live engine config (memory/threads/http/cache) so UI
+            # changes take effect on this persistent connection per query.
+            apply_runtime_pragmas(con, engine_config)
+
             # Profiling PRAGMAs are connection-level state.  Under concurrent
             # queries the last SET wins — one query's profile may land in the
             # wrong file.  This is acceptable: profiling is best-effort

{supertable-2.2.4 → supertable-2.2.5}/supertable/engine/engine_common.py

@@ -140,7 +140,11 @@ def _parse_memory_limit_mb(value: str) -> Optional[int]:
     return int(amount)
 
 
-def _derive_thread_count(memory_limit_str: str, fallback: int = 2) -> int:
+def _derive_thread_count(
+        memory_limit_str: str,
+        fallback: int = 2,
+        multiplier: Optional[float] = None,
+) -> int:
     """Derive a safe DuckDB thread count for remote parquet reads.
 
     DuckDB uses synchronous IO: each thread can make at most one HTTP/S3
@@ -170,9 +174,10 @@ def _derive_thread_count(memory_limit_str: str, fallback: int = 2) -> int:
         return fallback
 
     cpu_count = _os.cpu_count() or fallback
-    multiplier = settings.SUPERTABLE_DUCKDB_IO_MULTIPLIER
+    if multiplier is None:
+        multiplier = settings.SUPERTABLE_DUCKDB_IO_MULTIPLIER
 
-    io_threads = cpu_count * multiplier
+    io_threads = int(cpu_count * multiplier)
     # Safety ceiling: never allocate fewer than ~400 MB per thread.
     memory_floor = max(1, mb // 400)
     result = min(io_threads, memory_floor)
@@ -625,6 +630,56 @@ def init_connection(
         pass
 
 
+def apply_runtime_pragmas(con: duckdb.DuckDBPyConnection, cfg) -> None:
+    """Re-apply the session-settable DuckDB pragmas from a live engine config.
+
+    DuckDB Lite/Pro reuse a persistent connection, so settings applied once at
+    ``init_connection`` time would otherwise freeze for the connection's life.
+    Calling this immediately before each query makes the org's engine config
+    (memory limit, thread count, HTTP timeout, external file cache) take effect
+    live — the connection adopts the latest values without being torn down.
+
+    ``cfg`` is an ``EngineRuntimeConfig``.  When None the connection keeps
+    whatever ``init_connection`` applied (settings-based), so callers without a
+    resolved config are unaffected.  Every pragma is best-effort: httpfs-only
+    settings silently no-op on connections that have not loaded httpfs.
+    """
+    if cfg is None:
+        return
+
+    memory_limit = (cfg.duckdb_memory_limit or "").strip() or "1GB"
+    try:
+        con.execute(f"PRAGMA memory_limit='{memory_limit}';")
+    except Exception:
+        pass
+
+    # Explicit thread count wins; otherwise derive from the live memory limit
+    # and IO multiplier (same formula as init_connection).
+    if cfg.duckdb_threads is not None:
+        thread_count = cfg.duckdb_threads
+    else:
+        thread_count = _derive_thread_count(memory_limit, multiplier=cfg.duckdb_io_multiplier)
+    try:
+        con.execute(f"SET threads={thread_count};")
+    except Exception:
+        pass
+
+    # httpfs settings — only effective once httpfs is loaded (remote reads).
+    if cfg.duckdb_http_timeout is not None:
+        try:
+            con.execute(f"SET http_timeout={int(cfg.duckdb_http_timeout)};")
+        except Exception:
+            pass
+
+    cache_size = (cfg.duckdb_external_cache_size or "").strip()
+    if cache_size:
+        try:
+            con.execute("SET enable_external_file_cache=true;")
+            con.execute(f"SET external_file_cache_max_size='{sanitize_sql_string(cache_size)}';")
+        except Exception:
+            pass
+
+
 # =========================================================
 # RBAC view creation
 # =========================================================

supertable-2.2.5/supertable/engine/engine_config.py

@@ -0,0 +1,153 @@
+# route: supertable.engine.engine_config
+"""
+Engine runtime configuration — single source of truth for resolution.
+
+Engine config is stored once per organization at **system scope**
+(``supertable:{org}:system:config:engine``).  It is global to the org, not
+per-supertable, and is resolved **live on every query** (no caching) so that
+changes made in the UI take effect on the next query without a restart or a
+connection reset.
+
+Resolution precedence, per field::
+
+    Redis (org system config)  →  environment variable  →  built-in default
+
+Two entry points share that precedence so the engine and the UI never disagree:
+
+  * :func:`resolve_engine_config`            → typed :class:`EngineRuntimeConfig`,
+                                                consumed by the executor and the
+                                                DuckDB leaf executors.
+  * :func:`resolve_engine_config_provenance` → per-field ``{value, source,
+                                                env_var, default}`` dict for the
+                                                API / UI to display where each
+                                                effective value came from.
+
+Built-in defaults mirror the :mod:`supertable.config.settings` dataclass
+defaults, so behaviour is identical whether a value flows through ``settings``
+(no Redis override) or through this resolver.
+"""
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Any, Dict, Optional, Tuple
+
+
+# Canonical field spec: key → (env var, built-in default as string).
+# Defaults mirror the settings dataclass defaults in
+# supertable/config/settings.py — keep the two in sync.
+_FIELD_SPEC: Dict[str, Tuple[str, str]] = {
+    "engine_lite_max_bytes":      ("SUPERTABLE_ENGINE_LITE_MAX_BYTES",      str(100 * 1024 * 1024)),
+    "engine_spark_min_bytes":     ("SUPERTABLE_ENGINE_SPARK_MIN_BYTES",     str(10 * 1024 * 1024 * 1024)),
+    "engine_freshness_sec":       ("SUPERTABLE_ENGINE_FRESHNESS_SEC",       "300"),
+    "duckdb_memory_limit":        ("SUPERTABLE_DUCKDB_MEMORY_LIMIT",        "1GB"),
+    "duckdb_io_multiplier":       ("SUPERTABLE_DUCKDB_IO_MULTIPLIER",       "3"),
+    "duckdb_threads":             ("SUPERTABLE_DUCKDB_THREADS",             ""),
+    "duckdb_http_timeout":        ("SUPERTABLE_DUCKDB_HTTP_TIMEOUT",        ""),
+    "duckdb_external_cache_size": ("SUPERTABLE_DUCKDB_EXTERNAL_CACHE_SIZE", "5GB"),
+}
+
+# Public tuple of recognised field names (whitelist for callers/tests).
+ENGINE_CONFIG_FIELDS: Tuple[str, ...] = tuple(_FIELD_SPEC.keys())
+
+
+@dataclass(frozen=True)
+class EngineRuntimeConfig:
+    """Effective, typed engine configuration for a single resolution."""
+
+    # auto-pick thresholds
+    engine_lite_max_bytes: int
+    engine_spark_min_bytes: int
+    engine_freshness_sec: int
+
+    # duckdb session pragmas (re-applied live per query)
+    duckdb_memory_limit: str             # e.g. "1GB"
+    duckdb_io_multiplier: float
+    duckdb_threads: Optional[int]        # None → auto-derive from memory limit
+    duckdb_http_timeout: Optional[int]   # seconds; None → leave DuckDB default
+    duckdb_external_cache_size: str      # "" → external file cache disabled
+
+
+def _redis_cfg(org: str, catalog: Optional[Any]) -> Dict[str, Any]:
+    """Read the stored engine config for ``org`` (best effort).
+
+    Returns an empty dict when no catalog is supplied or Redis is unreachable,
+    so the resolver degrades cleanly to env vars + defaults.
+    """
+    if not (org and catalog is not None):
+        return {}
+    try:
+        return catalog.get_engine_config(org) or {}
+    except Exception:
+        return {}
+
+
+def _effective(redis_cfg: Dict[str, Any], key: str) -> Tuple[str, str]:
+    """Return ``(value, source)`` for one field. source ∈ redis|env|default."""
+    env_var, default = _FIELD_SPEC[key]
+    redis_val = redis_cfg.get(key)
+    if redis_val is not None and str(redis_val).strip() != "":
+        return str(redis_val).strip(), "redis"
+    env_val = (os.getenv(env_var) or "").strip()
+    if env_val:
+        return env_val, "env"
+    return default, "default"
+
+
+def _to_int(s: Any, fallback: int) -> int:
+    try:
+        return int(str(s).strip())
+    except (TypeError, ValueError):
+        return fallback
+
+
+def _to_float(s: Any, fallback: float) -> float:
+    try:
+        return float(str(s).strip())
+    except (TypeError, ValueError):
+        return fallback
+
+
+def resolve_engine_config(org: str, catalog: Optional[Any] = None) -> EngineRuntimeConfig:
+    """Resolve effective engine config for ``org`` (Redis → env → default).
+
+    ``catalog`` is an optional ``RedisCatalog``.  When it is None or Redis is
+    unreachable the resolver falls back to environment variables and built-in
+    defaults, which keeps the engine usable without Redis (e.g. in unit tests).
+    """
+    cfg = _redis_cfg(org, catalog)
+    val = {k: _effective(cfg, k)[0] for k in _FIELD_SPEC}
+
+    threads = _to_int(val["duckdb_threads"], 0) if val["duckdb_threads"] else 0
+    http = _to_int(val["duckdb_http_timeout"], 0) if val["duckdb_http_timeout"] else 0
+
+    return EngineRuntimeConfig(
+        engine_lite_max_bytes=_to_int(val["engine_lite_max_bytes"], 100 * 1024 * 1024),
+        engine_spark_min_bytes=_to_int(val["engine_spark_min_bytes"], 10 * 1024 * 1024 * 1024),
+        engine_freshness_sec=_to_int(val["engine_freshness_sec"], 300),
+        duckdb_memory_limit=val["duckdb_memory_limit"].strip() or "1GB",
+        duckdb_io_multiplier=_to_float(val["duckdb_io_multiplier"], 3.0),
+        duckdb_threads=(threads if threads > 0 else None),
+        duckdb_http_timeout=(http if http > 0 else None),
+        duckdb_external_cache_size=val["duckdb_external_cache_size"].strip(),
+    )
+
+
+def resolve_engine_config_provenance(org: str, catalog: Optional[Any] = None) -> Dict[str, Any]:
+    """Per-field provenance for the UI: ``{value, source, env_var, default}``.
+
+    ``source`` is one of ``redis`` / ``env`` / ``default``.  The extra
+    ``modified_ms`` key carries the last-write timestamp (or None).
+    """
+    cfg = _redis_cfg(org, catalog)
+    result: Dict[str, Any] = {}
+    for key, (env_var, default) in _FIELD_SPEC.items():
+        value, source = _effective(cfg, key)
+        result[key] = {
+            "value": value,
+            "source": source,
+            "env_var": env_var,
+            "default": default,
+        }
+    result["modified_ms"] = cfg.get("modified_ms")
+    return result

{supertable-2.2.4 → supertable-2.2.5}/supertable/engine/executor.py

@@ -16,9 +16,9 @@ from supertable.utils.sql_parser import SQLParser
 from supertable.engine.engine_enum import Engine
 from supertable.engine.duckdb_lite import DuckDBLite
 from supertable.engine.duckdb_pro import DuckDBPro
+from supertable.engine.engine_config import resolve_engine_config, EngineRuntimeConfig
 from supertable.data_classes import Reflection
 from supertable.config.defaults import logger
-from supertable.config.settings import settings
 
 
 # Module-level singleton for the pro executor so the persistent
@@ -46,8 +46,23 @@ class Executor:
         self.organization = organization
         self.lite_exec = DuckDBLite(storage=storage)
         self.spark_exec = None
+        self._catalog = None  # lazily created RedisCatalog for live config reads
 
-    def _auto_pick(self, reflection: Reflection) -> Engine:
+    def _get_catalog(self):
+        """Lazily create a RedisCatalog for live engine-config reads.
+
+        Returns None when Redis is unreachable so config resolution degrades to
+        environment variables and built-in defaults instead of failing a query.
+        """
+        if self._catalog is None:
+            try:
+                from supertable.redis_catalog import RedisCatalog
+                self._catalog = RedisCatalog()
+            except Exception:
+                self._catalog = False  # sentinel: construction failed, do not retry
+        return self._catalog or None
+
+    def _auto_pick(self, reflection: Reflection, cfg: EngineRuntimeConfig) -> Engine:
         """Select the best engine based on data size and freshness.
 
         Decision matrix (all thresholds configurable via env vars):
@@ -74,21 +89,10 @@ class Executor:
         """
         bytes_total = reflection.reflection_bytes
 
-        # --- thresholds (env-configurable) ---
-        try:
-            lite_max = settings.SUPERTABLE_ENGINE_LITE_MAX_BYTES
-        except ValueError:
-            lite_max = 100 * 1024 * 1024  # 100 MB
-
-        try:
-            spark_min = settings.SUPERTABLE_ENGINE_SPARK_MIN_BYTES
-        except ValueError:
-            spark_min = 10 * 1024 * 1024 * 1024  # 10 GB
-
-        try:
-            freshness_threshold_s = settings.SUPERTABLE_ENGINE_FRESHNESS_SEC
-        except ValueError:
-            freshness_threshold_s = 300  # 5 min
+        # --- thresholds (resolved live from org system config) ---
+        lite_max = cfg.engine_lite_max_bytes
+        spark_min = cfg.engine_spark_min_bytes
+        freshness_threshold_s = cfg.engine_freshness_sec
 
         # --- freshness: how long ago was the most recent snapshot updated ---
         if reflection.freshness_ms > 0:
@@ -138,7 +142,11 @@ class Executor:
         plan_stats: PlanStats,
         log_prefix: str,
     ) -> Tuple[pd.DataFrame, str]:
-        chosen = engine if engine != Engine.AUTO else self._auto_pick(reflection)
+        # Resolve engine config live (Redis → env → default) for this query so
+        # UI changes take effect immediately without restart or cache.
+        cfg = resolve_engine_config(self.organization, self._get_catalog())
+
+        chosen = engine if engine != Engine.AUTO else self._auto_pick(reflection, cfg)
 
         def timer_capture(evt: str):
             timer.capture_and_reset_timing(evt)
@@ -150,6 +158,7 @@ class Executor:
                 query_manager=query_manager,
                 timer_capture=timer_capture,
                 log_prefix=log_prefix,
+                engine_config=cfg,
             )
             used = "duckdb_lite"
 
@@ -161,6 +170,7 @@ class Executor:
                 query_manager=query_manager,
                 timer_capture=timer_capture,
                 log_prefix=log_prefix,
+                engine_config=cfg,
             )
             used = "duckdb_pro"
 

{supertable-2.2.4 → supertable-2.2.5}/supertable/redis_catalog.py

@@ -1525,6 +1525,20 @@ return 1
             return []
 
 
+    def delete_spark_cluster(self, org: str, cluster_id: str) -> bool:
+        """Remove a registered Spark Thrift cluster. Returns True if it existed."""
+        if not (org and cluster_id):
+            return False
+        try:
+            removed = self.r.hdel(RK.spark_thrifts(org), cluster_id)
+            if removed:
+                logger.info(f"[redis-catalog] spark thrift deleted: {org}/{cluster_id}")
+            return bool(removed)
+        except redis.RedisError as e:
+            logger.error(f"[redis-catalog] delete_spark_cluster error: {e}")
+            return False
+
+
     def select_spark_cluster(self, org: str, job_bytes: int, force: bool = False) -> Optional[Dict[str, Any]]:
         """
         Select the best active Spark Thrift cluster for a job of the given size.
@@ -1669,18 +1683,20 @@ return 1
     def set_engine_config(
             self,
             org: str,
-            sup: str,
             config: Dict[str, Any],
     ) -> bool:
         """Store engine runtime configuration (DuckDB + auto-pick thresholds).
 
+        Org-level system scope: one engine config per organization, applied
+        globally across all supertables (not per-supertable).
+
         The config dict is stored as a JSON string.  Only recognised fields
         (see ENGINE_CONFIG_FIELDS) are persisted — unknown keys are silently
         dropped to prevent injection of arbitrary settings.
 
         Existing config is fully replaced (last-write-wins).
         """
-        if not (org and sup):
+        if not org:
             return False
         try:
             # Whitelist recognised fields only.
@@ -1691,7 +1707,7 @@ return 1
             }
             doc["modified_ms"] = _now_ms()
             self.r.set(
-                RK.config_engine(org, sup),
+                RK.config_engine(org),
                 json.dumps(doc, default=str),
             )
             return True
@@ -1702,16 +1718,15 @@ return 1
     def get_engine_config(
             self,
             org: str,
-            sup: str,
     ) -> Optional[Dict[str, Any]]:
-        """Retrieve engine runtime configuration.
+        """Retrieve engine runtime configuration (org-level system scope).
 
-        Returns None if no config has been stored for this tenant.
+        Returns None if no config has been stored for this organization.
         """
-        if not (org and sup):
+        if not org:
             return None
         try:
-            raw = self.r.get(RK.config_engine(org, sup))
+            raw = self.r.get(RK.config_engine(org))
             if raw:
                 return json.loads(raw)
         except (redis.RedisError, json.JSONDecodeError) as e:

{supertable-2.2.4 → supertable-2.2.5}/supertable/redis_keys.py

@@ -38,6 +38,7 @@ Hierarchy (v2)
           spark:
             thrifts                                  HASH    Spark Thrift clusters
             plugs                                    HASH    Spark Plug runtimes
+          config:engine                              STRING  engine runtime config
         lakes:                                      ← user-data sentinel
           {sup}:                                    ← supertable scope
             meta:
@@ -55,7 +56,6 @@ Hierarchy (v2)
                   pipes:
                     index                            SET     pipe names
                     doc:{pipe_name}                  STRING  pipe definition
-            config:engine                            STRING  engine runtime config
             lock:
               leaf:
                 doc:{simple}                         STRING  per-table lock token
@@ -572,12 +572,9 @@ def pipe_pattern(org: str, sup: str, staging_name: str) -> str:
 
 # --- Engine config --------------------------------------------------------- #
 
-def config_engine(org: str, sup: str) -> str:
-    """Engine runtime config (STRING)."""
-    return (
-        f"{SUPERTABLE_PREFIX}:{_safe('org', org)}:{LAKES_SCOPE}"
-        f":{_safe('sup', sup)}:config:engine"
-    )
+def config_engine(org: str) -> str:
+    """Engine runtime config (STRING). Org-level system scope (global, not per-supertable)."""
+    return f"{SUPERTABLE_PREFIX}:{_safe('org', org)}:{SYSTEM_SCOPE}:config:engine"
 
 
 # --- Locks ----------------------------------------------------------------- #

{supertable-2.2.4 → supertable-2.2.5}/supertable/tests/test_redis_key_prefix.py

@@ -70,6 +70,7 @@ def _all_helpers() -> list[tuple[str, str, str]]:
         ("share_index",                  RK.share_index(ORG),                  f"{sys_pre}:shares:index"),
         ("spark_thrifts",                RK.spark_thrifts(ORG),                f"{sys_pre}:spark:thrifts"),
         ("spark_plugs",                  RK.spark_plugs(ORG),                  f"{sys_pre}:spark:plugs"),
+        ("config_engine",                RK.config_engine(ORG),                f"{sys_pre}:config:engine"),
 
         # ---- Lakes scope (per-org supertable enumeration) ---------------
         ("lakes_scope",                  RK.lakes_scope(ORG),                  f"supertable:{ORG}:lakes"),
@@ -95,9 +96,6 @@ def _all_helpers() -> list[tuple[str, str, str]]:
         ("pipe_doc",                     RK.pipe_doc(ORG, SUP, STAGING, PIPE), f"{lake_pre}:meta:staging:doc:{STAGING}:pipes:doc:{PIPE}"),
         ("pipe_pattern",                 RK.pipe_pattern(ORG, SUP, STAGING),   f"{lake_pre}:meta:staging:doc:{STAGING}:pipes:doc:*"),
 
-        # ---- Config -----------------------------------------------------
-        ("config_engine",                RK.config_engine(ORG, SUP),           f"{lake_pre}:config:engine"),
-
         # ---- Locks ------------------------------------------------------
         ("lock_leaf",                    RK.lock_leaf(ORG, SUP, SIMPLE),       f"{lake_pre}:lock:leaf:doc:{SIMPLE}"),
         ("lock_leaf_pattern",            RK.lock_leaf_pattern(ORG, SUP),       f"{lake_pre}:lock:leaf:doc:*"),

{supertable-2.2.4 → supertable-2.2.5/supertable.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supertable
-Version: 2.2.4
+Version: 2.2.5
 Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
 Author: Levente Kupas
 Author-email: Levente Kupas <lkupas@kladnasoft.com>

{supertable-2.2.4 → supertable-2.2.5}/supertable.egg-info/SOURCES.txt

@@ -99,6 +99,7 @@ supertable/engine/data_estimator.py
 supertable/engine/duckdb_lite.py
 supertable/engine/duckdb_pro.py
 supertable/engine/engine_common.py
+supertable/engine/engine_config.py
 supertable/engine/engine_enum.py
 supertable/engine/executor.py
 supertable/engine/plan_stats.py