supertable 2.2.5__tar.gz → 2.2.6__tar.gz

{supertable-2.2.5/supertable.egg-info → supertable-2.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supertable
-Version: 2.2.5
+Version: 2.2.6
 Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
 Author: Levente Kupas
 Author-email: Levente Kupas <lkupas@kladnasoft.com>

{supertable-2.2.5 → supertable-2.2.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "supertable"
-version = "2.2.5"
+version = "2.2.6"
 description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
 readme = "README.md"
 requires-python = ">=3.10"

{supertable-2.2.5 → supertable-2.2.6}/setup.py

@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
 
 setup(
     name="supertable",
-    version="2.2.5",
+    version="2.2.6",
     description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
     long_description=long_description,
     long_description_content_type="text/markdown",

{supertable-2.2.5 → supertable-2.2.6}/supertable/__init__.py

@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
 project documentation for the full API surface.
 """
 
-__version__ = "2.2.5"
+__version__ = "2.2.6"
 
 # Re-export the core public surface so users can do ``from supertable import …``
 # instead of remembering submodule paths.

supertable-2.2.6/supertable/engine/engine_config.py

@@ -0,0 +1,228 @@
+# route: supertable.engine.engine_config
+"""
+Engine runtime configuration — single source of truth for resolution.
+
+Engine config is stored once per organization at **system scope**
+(``supertable:{org}:system:engine:duckdb``).  It is global to the org, not
+per-supertable, and is resolved **live on every query** (no caching) so that
+changes made in the UI take effect on the next query without a restart or a
+connection reset.
+
+Two kinds of settings live in that document:
+
+  * **Shared auto-pick thresholds** (one value, used by the router):
+    ``engine_lite_max_bytes``, ``engine_spark_min_bytes``,
+    ``engine_freshness_sec``.  These describe the *boundaries* between engines
+    so they cannot be per-engine.
+  * **Per-engine DuckDB runtime pragmas** (separate values for Lite and Pro):
+    ``duckdb_memory_limit``, ``duckdb_io_multiplier``, ``duckdb_threads``,
+    ``duckdb_http_timeout``, ``duckdb_external_cache_size``.  Each DuckDB engine
+    carries its own copy under a ``"lite"`` / ``"pro"`` section, so tuning Lite
+    never changes Pro and vice-versa.
+
+Stored document shape::
+
+    {
+      "engine_lite_max_bytes": "104857600",
+      "engine_spark_min_bytes": "10737418240",
+      "engine_freshness_sec":   "300",
+      "lite": {"duckdb_memory_limit": "1GB", ...},
+      "pro":  {"duckdb_memory_limit": "8GB", ...},
+      "modified_ms": 1750000000000
+    }
+
+Resolution precedence, per field::
+
+    Redis (org system config)  →  environment variable  →  built-in default
+
+The environment variable is a single global fallback shared by both engines;
+per-engine differentiation therefore only happens once a value is stored in
+Redis (via the UI).
+
+Entry points (all share that precedence so engine and UI never disagree):
+
+  * :func:`resolve_engine_configs`           → ``{"lite": cfg, "pro": cfg}`` in a
+                                                single Redis read; used by the
+                                                executor.
+  * :func:`resolve_engine_config`            → typed :class:`EngineRuntimeConfig`
+                                                for one engine (convenience).
+  * :func:`resolve_engine_config_provenance` → ``{shared, lite, pro,
+                                                modified_ms}`` per-field
+                                                ``{value, source, env_var,
+                                                default}`` for the API / UI.
+
+Built-in defaults mirror the :mod:`supertable.config.settings` dataclass
+defaults, so behaviour is identical whether a value flows through ``settings``
+(no Redis override) or through this resolver.
+"""
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Any, Dict, Optional, Tuple
+
+
+# Shared auto-pick thresholds: key → (env var, built-in default as string).
+_SHARED_SPEC: Dict[str, Tuple[str, str]] = {
+    "engine_lite_max_bytes":  ("SUPERTABLE_ENGINE_LITE_MAX_BYTES",  str(100 * 1024 * 1024)),
+    "engine_spark_min_bytes": ("SUPERTABLE_ENGINE_SPARK_MIN_BYTES", str(10 * 1024 * 1024 * 1024)),
+    "engine_freshness_sec":   ("SUPERTABLE_ENGINE_FRESHNESS_SEC",   "300"),
+}
+
+# Per-engine DuckDB runtime pragmas: key → (env var, built-in default as string).
+# Defaults mirror the settings dataclass defaults in supertable/config/settings.py.
+_DUCKDB_SPEC: Dict[str, Tuple[str, str]] = {
+    "duckdb_memory_limit":        ("SUPERTABLE_DUCKDB_MEMORY_LIMIT",        "1GB"),
+    "duckdb_io_multiplier":       ("SUPERTABLE_DUCKDB_IO_MULTIPLIER",       "3"),
+    "duckdb_threads":             ("SUPERTABLE_DUCKDB_THREADS",             ""),
+    "duckdb_http_timeout":        ("SUPERTABLE_DUCKDB_HTTP_TIMEOUT",        ""),
+    "duckdb_external_cache_size": ("SUPERTABLE_DUCKDB_EXTERNAL_CACHE_SIZE", "5GB"),
+}
+
+# The two DuckDB engines that each carry their own runtime section.
+DUCKDB_ENGINES: Tuple[str, ...] = ("lite", "pro")
+
+# Public whitelists (for callers / catalog / tests).
+SHARED_CONFIG_FIELDS: Tuple[str, ...] = tuple(_SHARED_SPEC.keys())
+DUCKDB_CONFIG_FIELDS: Tuple[str, ...] = tuple(_DUCKDB_SPEC.keys())
+ENGINE_CONFIG_FIELDS: Tuple[str, ...] = SHARED_CONFIG_FIELDS + DUCKDB_CONFIG_FIELDS
+
+
+@dataclass(frozen=True)
+class EngineRuntimeConfig:
+    """Effective, typed config for one DuckDB engine resolution.
+
+    Carries the shared auto-pick thresholds (identical across engines) plus the
+    DuckDB session pragmas for the engine it was resolved for.
+    """
+
+    # auto-pick thresholds (shared)
+    engine_lite_max_bytes: int
+    engine_spark_min_bytes: int
+    engine_freshness_sec: int
+
+    # duckdb session pragmas (per-engine, re-applied live per query)
+    duckdb_memory_limit: str             # e.g. "1GB"
+    duckdb_io_multiplier: float
+    duckdb_threads: Optional[int]        # None → auto-derive from memory limit
+    duckdb_http_timeout: Optional[int]   # seconds; None → leave DuckDB default
+    duckdb_external_cache_size: str      # "" → external file cache disabled
+
+
+def _redis_cfg(org: str, catalog: Optional[Any]) -> Dict[str, Any]:
+    """Read the stored engine config for ``org`` (best effort).
+
+    Returns an empty dict when no catalog is supplied or Redis is unreachable,
+    so the resolver degrades cleanly to env vars + defaults.
+    """
+    if not (org and catalog is not None):
+        return {}
+    try:
+        return catalog.get_engine_config(org) or {}
+    except Exception:
+        return {}
+
+
+def _effective(stored: Dict[str, Any], key: str, spec: Dict[str, Tuple[str, str]]) -> Tuple[str, str]:
+    """Return ``(value, source)`` for one field. source ∈ redis|env|default.
+
+    ``stored`` is the dict the value is looked up in directly: the top-level
+    document for shared fields, or the ``lite`` / ``pro`` sub-section for
+    per-engine fields.
+    """
+    env_var, default = spec[key]
+    redis_val = stored.get(key) if isinstance(stored, dict) else None
+    if redis_val is not None and str(redis_val).strip() != "":
+        return str(redis_val).strip(), "redis"
+    env_val = (os.getenv(env_var) or "").strip()
+    if env_val:
+        return env_val, "env"
+    return default, "default"
+
+
+def _to_int(s: Any, fallback: int) -> int:
+    try:
+        return int(str(s).strip())
+    except (TypeError, ValueError):
+        return fallback
+
+
+def _to_float(s: Any, fallback: float) -> float:
+    try:
+        return float(str(s).strip())
+    except (TypeError, ValueError):
+        return fallback
+
+
+def _build_runtime(redis_cfg: Dict[str, Any], engine: str) -> EngineRuntimeConfig:
+    """Build the typed config for ``engine`` ("lite"/"pro") from a stored doc."""
+    section = redis_cfg.get(engine) if isinstance(redis_cfg.get(engine), dict) else {}
+
+    sv = {k: _effective(redis_cfg, k, _SHARED_SPEC)[0] for k in _SHARED_SPEC}
+    dv = {k: _effective(section, k, _DUCKDB_SPEC)[0] for k in _DUCKDB_SPEC}
+
+    threads = _to_int(dv["duckdb_threads"], 0) if dv["duckdb_threads"] else 0
+    http = _to_int(dv["duckdb_http_timeout"], 0) if dv["duckdb_http_timeout"] else 0
+
+    return EngineRuntimeConfig(
+        engine_lite_max_bytes=_to_int(sv["engine_lite_max_bytes"], 100 * 1024 * 1024),
+        engine_spark_min_bytes=_to_int(sv["engine_spark_min_bytes"], 10 * 1024 * 1024 * 1024),
+        engine_freshness_sec=_to_int(sv["engine_freshness_sec"], 300),
+        duckdb_memory_limit=dv["duckdb_memory_limit"].strip() or "1GB",
+        duckdb_io_multiplier=_to_float(dv["duckdb_io_multiplier"], 3.0),
+        duckdb_threads=(threads if threads > 0 else None),
+        duckdb_http_timeout=(http if http > 0 else None),
+        duckdb_external_cache_size=dv["duckdb_external_cache_size"].strip(),
+    )
+
+
+def resolve_engine_configs(org: str, catalog: Optional[Any] = None) -> Dict[str, EngineRuntimeConfig]:
+    """Resolve both DuckDB engines in a single Redis read.
+
+    Returns ``{"lite": EngineRuntimeConfig, "pro": EngineRuntimeConfig}``.  The
+    shared thresholds are identical in both; only the DuckDB pragmas differ.
+    """
+    cfg = _redis_cfg(org, catalog)
+    return {e: _build_runtime(cfg, e) for e in DUCKDB_ENGINES}
+
+
+def resolve_engine_config(org: str, catalog: Optional[Any] = None, engine: str = "lite") -> EngineRuntimeConfig:
+    """Resolve effective config for a single ``engine`` (Redis → env → default).
+
+    ``catalog`` is an optional ``RedisCatalog``.  When it is None or Redis is
+    unreachable the resolver falls back to environment variables and built-in
+    defaults, which keeps the engine usable without Redis (e.g. in unit tests).
+    """
+    engine = engine if engine in DUCKDB_ENGINES else "lite"
+    return _build_runtime(_redis_cfg(org, catalog), engine)
+
+
+def resolve_engine_config_provenance(org: str, catalog: Optional[Any] = None) -> Dict[str, Any]:
+    """Per-field provenance for the UI.
+
+    Shape::
+
+        {
+          "shared": {field: {value, source, env_var, default}, ...},
+          "lite":   {field: {value, source, env_var, default}, ...},
+          "pro":    {field: {value, source, env_var, default}, ...},
+          "modified_ms": <int|None>,
+        }
+
+    ``source`` is one of ``redis`` / ``env`` / ``default``.
+    """
+    cfg = _redis_cfg(org, catalog)
+
+    def _block(stored: Dict[str, Any], spec: Dict[str, Tuple[str, str]]) -> Dict[str, Any]:
+        out: Dict[str, Any] = {}
+        for key, (env_var, default) in spec.items():
+            value, source = _effective(stored, key, spec)
+            out[key] = {"value": value, "source": source, "env_var": env_var, "default": default}
+        return out
+
+    result: Dict[str, Any] = {"shared": _block(cfg, _SHARED_SPEC)}
+    for engine in DUCKDB_ENGINES:
+        section = cfg.get(engine) if isinstance(cfg.get(engine), dict) else {}
+        result[engine] = _block(section, _DUCKDB_SPEC)
+    result["modified_ms"] = cfg.get("modified_ms")
+    return result

{supertable-2.2.5 → supertable-2.2.6}/supertable/engine/executor.py

@@ -16,7 +16,7 @@ from supertable.utils.sql_parser import SQLParser
 from supertable.engine.engine_enum import Engine
 from supertable.engine.duckdb_lite import DuckDBLite
 from supertable.engine.duckdb_pro import DuckDBPro
-from supertable.engine.engine_config import resolve_engine_config, EngineRuntimeConfig
+from supertable.engine.engine_config import resolve_engine_configs, EngineRuntimeConfig
 from supertable.data_classes import Reflection
 from supertable.config.defaults import logger
 
@@ -143,10 +143,14 @@ class Executor:
         log_prefix: str,
     ) -> Tuple[pd.DataFrame, str]:
         # Resolve engine config live (Redis → env → default) for this query so
-        # UI changes take effect immediately without restart or cache.
-        cfg = resolve_engine_config(self.organization, self._get_catalog())
+        # UI changes take effect immediately without restart or cache.  Lite and
+        # Pro carry independent DuckDB pragmas; the shared auto-pick thresholds
+        # are identical in both, so either may drive the routing decision.
+        cfgs = resolve_engine_configs(self.organization, self._get_catalog())
+        lite_cfg = cfgs["lite"]
+        pro_cfg = cfgs["pro"]
 
-        chosen = engine if engine != Engine.AUTO else self._auto_pick(reflection, cfg)
+        chosen = engine if engine != Engine.AUTO else self._auto_pick(reflection, lite_cfg)
 
         def timer_capture(evt: str):
             timer.capture_and_reset_timing(evt)
@@ -158,7 +162,7 @@ class Executor:
                 query_manager=query_manager,
                 timer_capture=timer_capture,
                 log_prefix=log_prefix,
-                engine_config=cfg,
+                engine_config=lite_cfg,
             )
             used = "duckdb_lite"
 
@@ -170,7 +174,7 @@ class Executor:
                 query_manager=query_manager,
                 timer_capture=timer_capture,
                 log_prefix=log_prefix,
-                engine_config=cfg,
+                engine_config=pro_cfg,
             )
             used = "duckdb_pro"
 

{supertable-2.2.5 → supertable-2.2.6}/supertable/engine/spark_thrift.py

@@ -25,7 +25,7 @@ from supertable.engine.engine_common import (
     escape_parquet_path,
 )
 
-from urllib.parse import urlparse
+from urllib.parse import urlparse, unquote
 
 # Suppress verbose INFO logging from PyHive and Thrift libraries.
 # PyHive logs every SQL statement (CREATE VIEW, DROP VIEW, SET, etc.)
@@ -77,8 +77,14 @@ def _to_s3a_path(file_path: str) -> str:
 
     if file_path.startswith("http://") or file_path.startswith("https://"):
         parsed = urlparse(file_path)
-        # path is /bucket/key — strip the leading slash, split into bucket + key
-        path = parsed.path.lstrip("/")
+        # path is /bucket/key — strip the leading slash, split into bucket + key.
+        # urlparse leaves percent-escapes intact, and boto3 presigning encodes
+        # the key (Hive partition dirs like ``year=2006`` become ``year%3D2006``).
+        # Spark's S3A client expects the *raw* object key and re-encodes itself,
+        # so we must decode here — otherwise Ceph RGW, which matches keys
+        # literally, 404s on the ``%3D`` form. The presign query string is
+        # dropped on purpose: Spark authenticates with its own fs.s3a.* creds.
+        path = unquote(parsed.path).lstrip("/")
         if "/" in path:
             return f"s3a://{path}"
         # Degenerate case: only bucket, no key — return as-is

{supertable-2.2.5 → supertable-2.2.6}/supertable/redis_catalog.py

@@ -1468,7 +1468,7 @@ return 1
             return 0
 
     # ========================================================================= #
-    # Spark Thrift cluster management (org-scoped: supertable:{org}:system:spark:thrifts)
+    # Spark Thrift cluster management (org-scoped: supertable:{org}:system:engine:thrifts)
     # ========================================================================= #
 
     def register_spark_cluster(self, org: str, cluster_id: str, config: Dict[str, Any]) -> None:
@@ -1497,7 +1497,7 @@ return 1
             doc.setdefault("s3_enabled", True)
             doc["modified_ms"] = _now_ms()
             self.r.hset(
-                RK.spark_thrifts(org),
+                RK.engine_thrifts(org),
                 cluster_id,
                 json.dumps(doc, default=str),
             )
@@ -1512,7 +1512,7 @@ return 1
         if not org:
             return []
         try:
-            raw = self.r.hgetall(RK.spark_thrifts(org))
+            raw = self.r.hgetall(RK.engine_thrifts(org))
             clusters = []
             for _cid, data in raw.items():
                 try:
@@ -1530,7 +1530,7 @@ return 1
         if not (org and cluster_id):
             return False
         try:
-            removed = self.r.hdel(RK.spark_thrifts(org), cluster_id)
+            removed = self.r.hdel(RK.engine_thrifts(org), cluster_id)
             if removed:
                 logger.info(f"[redis-catalog] spark thrift deleted: {org}/{cluster_id}")
             return bool(removed)
@@ -1578,7 +1578,7 @@ return 1
         return candidates[0]
 
     # ========================================================================= #
-    # Spark Plug management (org-scoped: supertable:{org}:system:spark:plugs)
+    # Spark Plug management (org-scoped: supertable:{org}:system:engine:plugs)
     # ========================================================================= #
 
     def register_spark_plug(self, org: str, plug_id: str, config: Dict[str, Any]) -> None:
@@ -1604,7 +1604,7 @@ return 1
             doc.setdefault("webui_url", "")
             doc["modified_ms"] = _now_ms()
             self.r.hset(
-                RK.spark_plugs(org),
+                RK.engine_plugs(org),
                 plug_id,
                 json.dumps(doc, default=str),
             )
@@ -1669,45 +1669,53 @@ return 1
     # Canonical field names and their env-var counterparts.  Used by the
     # resolver in engine_common to fall back to os.getenv when a field
     # is absent from Redis.
-    ENGINE_CONFIG_FIELDS = {
-        "engine_lite_max_bytes":       "SUPERTABLE_ENGINE_LITE_MAX_BYTES",
-        "engine_spark_min_bytes":      "SUPERTABLE_ENGINE_SPARK_MIN_BYTES",
-        "engine_freshness_sec":        "SUPERTABLE_ENGINE_FRESHNESS_SEC",
-        "duckdb_memory_limit":         "SUPERTABLE_DUCKDB_MEMORY_LIMIT",
-        "duckdb_io_multiplier":        "SUPERTABLE_DUCKDB_IO_MULTIPLIER",
-        "duckdb_threads":              "SUPERTABLE_DUCKDB_THREADS",
-        "duckdb_http_timeout":         "SUPERTABLE_DUCKDB_HTTP_TIMEOUT",
-        "duckdb_external_cache_size":  "SUPERTABLE_DUCKDB_EXTERNAL_CACHE_SIZE",
-    }
+    # Per-engine DuckDB runtime pragmas (separate sections for "lite" / "pro").
+    DUCKDB_CONFIG_FIELDS = (
+        "duckdb_memory_limit",
+        "duckdb_io_multiplier",
+        "duckdb_threads",
+        "duckdb_http_timeout",
+        "duckdb_external_cache_size",
+    )
+    DUCKDB_ENGINES = ("lite", "pro")
 
     def set_engine_config(
             self,
             org: str,
+            engine: str,
             config: Dict[str, Any],
     ) -> bool:
-        """Store engine runtime configuration (DuckDB + auto-pick thresholds).
+        """Store DuckDB runtime configuration for one engine ("lite" / "pro").
 
-        Org-level system scope: one engine config per organization, applied
+        Org-level system scope: one engine document per organization, applied
         globally across all supertables (not per-supertable).
 
-        The config dict is stored as a JSON string.  Only recognised fields
-        (see ENGINE_CONFIG_FIELDS) are persisted — unknown keys are silently
-        dropped to prevent injection of arbitrary settings.
+        Only the requested engine's section is replaced; the other engine's
+        section and the shared auto-pick thresholds are preserved.  Within the
+        section it is a full replace (last-write-wins) — omitting a field clears
+        its Redis override so it falls back to env / default at read time.
 
-        Existing config is fully replaced (last-write-wins).
+        Only recognised DuckDB fields (see DUCKDB_CONFIG_FIELDS) are persisted —
+        unknown keys are silently dropped to prevent injection of arbitrary
+        settings.
         """
         if not org:
             return False
+        engine = (engine or "").strip().lower()
+        if engine not in self.DUCKDB_ENGINES:
+            return False
         try:
-            # Whitelist recognised fields only.
-            doc = {
+            doc = self.get_engine_config(org) or {}
+            # Whitelist recognised DuckDB fields only for this engine section.
+            section = {
                 k: config[k]
-                for k in self.ENGINE_CONFIG_FIELDS
+                for k in self.DUCKDB_CONFIG_FIELDS
                 if k in config and config[k] is not None and str(config[k]).strip() != ""
             }
+            doc[engine] = section
             doc["modified_ms"] = _now_ms()
             self.r.set(
-                RK.config_engine(org),
+                RK.engine_duckdb(org),
                 json.dumps(doc, default=str),
             )
             return True
@@ -1726,7 +1734,7 @@ return 1
         if not org:
             return None
         try:
-            raw = self.r.get(RK.config_engine(org))
+            raw = self.r.get(RK.engine_duckdb(org))
             if raw:
                 return json.loads(raw)
         except (redis.RedisError, json.JSONDecodeError) as e:

{supertable-2.2.5 → supertable-2.2.6}/supertable/redis_keys.py

@@ -35,10 +35,10 @@ Hierarchy (v2)
           shares:
             doc:{share_id}                           STRING  share definition
             index                                    SET     share IDs
-          spark:
+          engine:
             thrifts                                  HASH    Spark Thrift clusters
             plugs                                    HASH    Spark Plug runtimes
-          config:engine                              STRING  engine runtime config
+            duckdb                                   STRING  DuckDB runtime config
         lakes:                                      ← user-data sentinel
           {sup}:                                    ← supertable scope
             meta:
@@ -121,7 +121,7 @@ SUPERTABLE_PREFIX: str = "supertable"
 DATAISLAND_PREFIX: str = "dataisland"
 
 # Position-2 sentinel under ``supertable:{org}:`` for org-level platform
-# state (auth tokens, audit, shares, spark).
+# state (auth tokens, audit, shares, engine).
 SYSTEM_SCOPE: str = "system"
 
 # Position-2 sentinel under ``supertable:{org}:`` for user-supplied
@@ -394,16 +394,21 @@ def share_index(org: str) -> str:
     return f"{SUPERTABLE_PREFIX}:{_safe('org', org)}:{SYSTEM_SCOPE}:shares:index"
 
 
-# --- Org-level Spark ------------------------------------------------------- #
+# --- Org-level Engine ------------------------------------------------------ #
+#
+# All query-engine state lives under one ``system:engine:`` namespace: the
+# Spark Thrift cluster registry, the Spark Plug runtime registry, and the
+# DuckDB (Lite/Pro) runtime config document. Grouping them keeps every engine
+# knob the UI exposes under a single, discoverable path.
 
-def spark_thrifts(org: str) -> str:
+def engine_thrifts(org: str) -> str:
     """Spark Thrift cluster registry (HASH)."""
-    return f"{SUPERTABLE_PREFIX}:{_safe('org', org)}:{SYSTEM_SCOPE}:spark:thrifts"
+    return f"{SUPERTABLE_PREFIX}:{_safe('org', org)}:{SYSTEM_SCOPE}:engine:thrifts"
 
 
-def spark_plugs(org: str) -> str:
+def engine_plugs(org: str) -> str:
     """Spark Plug runtime registry (HASH)."""
-    return f"{SUPERTABLE_PREFIX}:{_safe('org', org)}:{SYSTEM_SCOPE}:spark:plugs"
+    return f"{SUPERTABLE_PREFIX}:{_safe('org', org)}:{SYSTEM_SCOPE}:engine:plugs"
 
 
 # =========================================================================== #
@@ -570,11 +575,15 @@ def pipe_pattern(org: str, sup: str, staging_name: str) -> str:
     )
 
 
-# --- Engine config --------------------------------------------------------- #
+# --- DuckDB engine config -------------------------------------------------- #
+
+def engine_duckdb(org: str) -> str:
+    """DuckDB (Lite/Pro) runtime config (STRING). Org-level system scope (global, not per-supertable).
 
-def config_engine(org: str) -> str:
-    """Engine runtime config (STRING). Org-level system scope (global, not per-supertable)."""
-    return f"{SUPERTABLE_PREFIX}:{_safe('org', org)}:{SYSTEM_SCOPE}:config:engine"
+    One document holds both engines' DuckDB pragmas plus the shared auto-pick
+    thresholds; see :mod:`supertable.engine.engine_config`.
+    """
+    return f"{SUPERTABLE_PREFIX}:{_safe('org', org)}:{SYSTEM_SCOPE}:engine:duckdb"
 
 
 # --- Locks ----------------------------------------------------------------- #

{supertable-2.2.5 → supertable-2.2.6}/supertable/tests/test_redis_key_prefix.py

@@ -68,9 +68,9 @@ def _all_helpers() -> list[tuple[str, str, str]]:
         ("audit_legal_hold",             RK.audit_legal_hold(ORG),             f"{sys_pre}:audit:legal_hold"),
         ("share_doc",                    RK.share_doc(ORG, SHARE),             f"{sys_pre}:shares:doc:{SHARE}"),
         ("share_index",                  RK.share_index(ORG),                  f"{sys_pre}:shares:index"),
-        ("spark_thrifts",                RK.spark_thrifts(ORG),                f"{sys_pre}:spark:thrifts"),
-        ("spark_plugs",                  RK.spark_plugs(ORG),                  f"{sys_pre}:spark:plugs"),
-        ("config_engine",                RK.config_engine(ORG),                f"{sys_pre}:config:engine"),
+        ("engine_thrifts",               RK.engine_thrifts(ORG),               f"{sys_pre}:engine:thrifts"),
+        ("engine_plugs",                 RK.engine_plugs(ORG),                 f"{sys_pre}:engine:plugs"),
+        ("engine_duckdb",                RK.engine_duckdb(ORG),                f"{sys_pre}:engine:duckdb"),
 
         # ---- Lakes scope (per-org supertable enumeration) ---------------
         ("lakes_scope",                  RK.lakes_scope(ORG),                  f"supertable:{ORG}:lakes"),

{supertable-2.2.5 → supertable-2.2.6/supertable.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supertable
-Version: 2.2.5
+Version: 2.2.6
 Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
 Author: Levente Kupas
 Author-email: Levente Kupas <lkupas@kladnasoft.com>