supertable 2.2.6__tar.gz → 2.2.8__tar.gz

{supertable-2.2.6/supertable.egg-info → supertable-2.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supertable
-Version: 2.2.6
+Version: 2.2.8
 Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
 Author: Levente Kupas
 Author-email: Levente Kupas <lkupas@kladnasoft.com>

{supertable-2.2.6 → supertable-2.2.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "supertable"
-version = "2.2.6"
+version = "2.2.8"
 description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
 readme = "README.md"
 requires-python = ">=3.10"

{supertable-2.2.6 → supertable-2.2.8}/setup.py

@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
 
 setup(
     name="supertable",
-    version="2.2.6",
+    version="2.2.8",
     description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
     long_description=long_description,
     long_description_content_type="text/markdown",

{supertable-2.2.6 → supertable-2.2.8}/supertable/__init__.py

@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
 project documentation for the full API surface.
 """
 
-__version__ = "2.2.6"
+__version__ = "2.2.8"
 
 # Re-export the core public surface so users can do ``from supertable import …``
 # instead of remembering submodule paths.

{supertable-2.2.6 → supertable-2.2.8}/supertable/config/settings.py

@@ -160,7 +160,7 @@ class Settings:
 
     # ── Engine Routing / Executor ────────────────────────────────────
     SUPERTABLE_ENGINE_LITE_MAX_BYTES: int = 100 * 1024 * 1024    # SUPERTABLE_ENGINE_LITE_MAX_BYTES (100 MB)
-    SUPERTABLE_ENGINE_SPARK_MIN_BYTES: int = 10 * 1024 * 1024 * 1024  # SUPERTABLE_ENGINE_SPARK_MIN_BYTES (10 GB)
+    SUPERTABLE_ENGINE_SPARK_MIN_BYTES: int = 0  # fallback only; 0 = active fleet's min_bytes drives Spark routing
     SUPERTABLE_ENGINE_FRESHNESS_SEC: int = 300     # SUPERTABLE_ENGINE_FRESHNESS_SEC
     SUPERTABLE_DEFAULT_ENGINE: str = "AUTO"        # SUPERTABLE_DEFAULT_ENGINE
 
@@ -410,7 +410,7 @@ def _build_settings() -> Settings:
 
         # ── Engine Routing ───────────────────────────────────────────
         SUPERTABLE_ENGINE_LITE_MAX_BYTES=_env_int("SUPERTABLE_ENGINE_LITE_MAX_BYTES", 100 * 1024 * 1024),
-        SUPERTABLE_ENGINE_SPARK_MIN_BYTES=_env_int("SUPERTABLE_ENGINE_SPARK_MIN_BYTES", 10 * 1024 * 1024 * 1024),
+        SUPERTABLE_ENGINE_SPARK_MIN_BYTES=_env_int("SUPERTABLE_ENGINE_SPARK_MIN_BYTES", 0),
         SUPERTABLE_ENGINE_FRESHNESS_SEC=_env_int("SUPERTABLE_ENGINE_FRESHNESS_SEC", 300),
         SUPERTABLE_DEFAULT_ENGINE=_env_str("SUPERTABLE_DEFAULT_ENGINE", "AUTO"),
 

{supertable-2.2.6 → supertable-2.2.8}/supertable/data_reader.py

@@ -44,10 +44,20 @@ class DataReader:
       - Execution to Executor (DuckDB/Spark)
     """
 
-    def __init__(self, super_name: str, organization: str, query: str):
+    def __init__(
+        self,
+        super_name: str,
+        organization: str,
+        query: str,
+        source: str = "sdk",
+    ):
         self.super_name = super_name
         self.organization = organization
         self.query = query
+        # Query origin surfaced in the reads monitoring tab. "sdk" is the
+        # default for direct SDK callers; the API/OData/MCP entry points
+        # pass "api"/"odata"/"mcp" so each query records where it came from.
+        self.source = source
 
         self.storage: StorageInterface = get_storage()
 
@@ -164,6 +174,9 @@ class DataReader:
                 current_meta_path="redis://meta/root",
                 query=parser.original_query,
             )
+            # Stamp the call origin so plan_extender records it on the read
+            # monitoring entry (defaults to "api" downstream if unset).
+            self.query_plan_manager.source_type = self.source
             self._log_ctx = f"[qid={self.query_plan_manager.query_id} qh={self.query_plan_manager.query_hash}] "
             self.query_plan_manager.original_table = ", ".join(t.simple_name for t in physical_tables) if physical_tables else ""
 
@@ -311,16 +324,25 @@ def query_sql(
         limit: int,
         engine: Any,
         role_name: str,
+        source: str = "sdk",
+        out: Optional[Dict[str, Any]] = None,
 ) -> Tuple[List[str], List[List[Any]], List[Dict[str, Any]]]:
     """
     Execute SQL query and return results in the format expected by MCP server.
     Returns: (columns, rows, columns_meta)
+
+    ``source`` tags the query origin on the read monitoring entry
+    (defaults to "sdk"; the MCP server passes "mcp"). When an ``out``
+    dict is supplied it is populated with ``query_id``/``query_hash`` so
+    the caller can correlate its own audit log to this read record.
     """
     # Safety guard: ensure a LIMIT is present so unbounded queries don't
     # overwhelm the MCP response payload.
     sql = _ensure_sql_limit(sql, default_limit=limit)
 
-    reader = DataReader(organization=organization, super_name=super_name, query=sql)
+    reader = DataReader(
+        organization=organization, super_name=super_name, query=sql, source=source,
+    )
 
     # Execute the query
     result_df, status, message = reader.execute(
@@ -329,6 +351,15 @@ def query_sql(
         with_scan=False,
     )
 
+    # Expose the query identity so the caller (e.g. the MCP audit log) can
+    # link back to this read's monitoring entry. Populated even on error,
+    # since the QueryPlanManager is created before execution.
+    if out is not None:
+        qpm = reader.query_plan_manager
+        if qpm is not None:
+            out["query_id"] = qpm.query_id
+            out["query_hash"] = qpm.query_hash
+
     if status == Status.ERROR:
         raise RuntimeError(f"Query execution failed: {message}")
 

{supertable-2.2.6 → supertable-2.2.8}/supertable/engine/engine_common.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 import hashlib
 import os
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 from urllib.parse import urlparse
 
 import duckdb
@@ -14,6 +14,7 @@ from sqlglot import exp
 from supertable.config.defaults import logger
 from supertable.config.settings import settings
 from supertable.config.homedir import get_app_home
+from supertable.engine.engine_config import normalize_memory_size
 
 
 # =========================================================
@@ -281,11 +282,15 @@ def configure_httpfs_and_s3(
         "true" if meta_cache_on else "false",
     )
 
-    # External file cache (DuckDB >= 1.3) — caches remote data blocks on local
-    # disk so repeated queries do not re-download the same row groups.
-    # Only enabled when SUPERTABLE_DUCKDB_EXTERNAL_CACHE_SIZE is set.
+    # External file cache — an in-memory cache of remote data blocks so
+    # repeated queries do not re-download the same row groups.  It is only
+    # enabled when a size cap can be enforced: DuckDB builds without
+    # external_file_cache_max_size (e.g. 1.5.x) cannot bound it, and an
+    # uncapped cache grows to memory_limit on the persistent connection.
+    # Without an enforceable cap we keep it OFF to protect memory.
     cache_size = settings.SUPERTABLE_DUCKDB_EXTERNAL_CACHE_SIZE
-    if cache_size:
+    can_cap = "external_file_cache_max_size" in supported
+    if cache_size and can_cap:
         set_if_supported("enable_external_file_cache", "true")
         set_if_supported("external_file_cache_max_size", f"'{cache_size}'")
         cache_dir_raw = settings.SUPERTABLE_DUCKDB_EXTERNAL_CACHE_DIR
@@ -301,6 +306,15 @@ def configure_httpfs_and_s3(
             + (f" size={cache_size}" if cache_size else "")
             + f", dir={cache_dir}"
         )
+    else:
+        # Uncappable (or disabled via empty size) — turn it off explicitly so
+        # the DuckDB 1.5.x default-on cache cannot accumulate in memory.
+        set_if_supported("enable_external_file_cache", "false")
+        if cache_size and not can_cap:
+            logger.info(
+                "[duckdb.cache] external file cache disabled: this DuckDB build "
+                "cannot cap it (no external_file_cache_max_size)"
+            )
 
 
 # =========================================================
@@ -554,6 +568,25 @@ def rewrite_query_with_hashed_tables(
 # Connection initialization
 # =========================================================
 
+def _external_file_cache_cappable(con: duckdb.DuckDBPyConnection) -> bool:
+    """True when this DuckDB build can bound the external file cache size.
+
+    DuckDB 1.5.x enables ``enable_external_file_cache`` by default but does
+    not expose ``external_file_cache_max_size``.  An enabled-but-uncapped
+    cache is held in memory (not on disk) and grows to ``memory_limit`` on
+    the long-lived persistent connection — a sustained-memory / OOM hazard.
+    When this returns False the cache is disabled outright rather than left
+    running unbounded.
+    """
+    try:
+        return bool(con.execute(
+            "SELECT 1 FROM duckdb_settings() "
+            "WHERE name = 'external_file_cache_max_size'"
+        ).fetchone())
+    except Exception:
+        return False
+
+
 def init_connection(
         con: duckdb.DuckDBPyConnection,
         temp_dir: str,
@@ -578,8 +611,18 @@ def init_connection(
     # Resolve memory limit.
     # Single env var SUPERTABLE_DUCKDB_MEMORY_LIMIT controls both executors.
     # The `memory_limit` argument is the caller's fallback when the env var is absent.
-    effective_memory_limit = settings.SUPERTABLE_DUCKDB_MEMORY_LIMIT or memory_limit
-    con.execute(f"PRAGMA memory_limit='{effective_memory_limit}';")
+    # normalize_memory_size guarantees a unit-suffixed value so DuckDB's parser
+    # never rejects a bare number (e.g. a UI-supplied "2" -> "2GB").
+    effective_memory_limit = normalize_memory_size(
+        settings.SUPERTABLE_DUCKDB_MEMORY_LIMIT or memory_limit, default="1GB"
+    )
+    try:
+        con.execute(f"PRAGMA memory_limit='{effective_memory_limit}';")
+    except Exception as e:
+        logger.warning(
+            f"[duckdb.init] memory_limit='{effective_memory_limit}' rejected: {e}; "
+            f"keeping DuckDB default"
+        )
 
     # Absolute temp path is required for DuckDB to actually spill to disk.
     # Prefer a path rooted under the app home (~/supertable), which is
@@ -605,6 +648,17 @@ def init_connection(
     except Exception:
         pass  # older DuckDB builds may not support this setting
 
+    # External file cache baseline.  DuckDB 1.5.x turns the cache ON by
+    # default but cannot cap it, so an uncapped in-memory cache accumulates
+    # remote data up to memory_limit on the persistent connection.  Disable
+    # it here when uncappable; configure_httpfs_and_s3 / apply_runtime_pragmas
+    # re-enable it (capped) only on builds that support a size cap.
+    if not _external_file_cache_cappable(con):
+        try:
+            con.execute("SET enable_external_file_cache=false;")
+        except Exception:
+            pass
+
     # Thread count.
     # If SUPERTABLE_DUCKDB_THREADS is set explicitly, honour it exactly.
     # Otherwise derive from the effective memory limit using the IO-thread
@@ -647,11 +701,14 @@ def apply_runtime_pragmas(con: duckdb.DuckDBPyConnection, cfg) -> None:
     if cfg is None:
         return
 
-    memory_limit = (cfg.duckdb_memory_limit or "").strip() or "1GB"
+    # normalize_memory_size guarantees a unit (UI sends a bare "2" -> "2GB");
+    # a rejected value is logged rather than silently swallowed so a bad
+    # config can never quietly leave the connection at the wrong limit.
+    memory_limit = normalize_memory_size(cfg.duckdb_memory_limit, default="1GB")
     try:
         con.execute(f"PRAGMA memory_limit='{memory_limit}';")
-    except Exception:
-        pass
+    except Exception as e:
+        logger.warning(f"[duckdb.pragma] memory_limit='{memory_limit}' rejected: {e}")
 
     # Explicit thread count wins; otherwise derive from the live memory limit
     # and IO multiplier (same formula as init_connection).
@@ -671,15 +728,296 @@ def apply_runtime_pragmas(con: duckdb.DuckDBPyConnection, cfg) -> None:
         except Exception:
             pass
 
-    cache_size = (cfg.duckdb_external_cache_size or "").strip()
-    if cache_size:
+    # External file cache: only run it when the size cap is enforceable.
+    # On DuckDB builds without external_file_cache_max_size (e.g. 1.5.x) an
+    # enabled cache is in-memory and unbounded — it grows to memory_limit on
+    # the persistent connection — so we disable it instead of running uncapped.
+    cache_size = normalize_memory_size(cfg.duckdb_external_cache_size, default="")
+    if cache_size and _external_file_cache_cappable(con):
         try:
             con.execute("SET enable_external_file_cache=true;")
-            con.execute(f"SET external_file_cache_max_size='{sanitize_sql_string(cache_size)}';")
+            con.execute(
+                f"SET external_file_cache_max_size='{sanitize_sql_string(cache_size)}';"
+            )
+        except Exception as e:
+            logger.warning(f"[duckdb.pragma] external file cache config failed: {e}")
+    else:
+        try:
+            con.execute("SET enable_external_file_cache=false;")
         except Exception:
             pass
 
 
+# =========================================================
+# Engine self-diagnostics (UI "Diagnose" button)
+# =========================================================
+
+def _filesystem_type(path: str) -> str:
+    """Best-effort filesystem type for ``path`` via /proc/mounts (Linux).
+
+    Used to warn when the spill directory is RAM-backed (tmpfs/ramfs), where
+    "spilling to disk" would actually consume memory instead of relieving it.
+    Returns "" when the type cannot be determined.
+    """
+    try:
+        target = os.path.abspath(path)
+        best_mp = ""
+        best_type = ""
+        with open("/proc/mounts", "r") as fh:
+            for line in fh:
+                parts = line.split()
+                if len(parts) < 3:
+                    continue
+                mount_point, fstype = parts[1], parts[2]
+                if (
+                    target == mount_point
+                    or target.startswith(mount_point.rstrip("/") + "/")
+                    or mount_point == "/"
+                ):
+                    if len(mount_point) >= len(best_mp):
+                        best_mp = mount_point
+                        best_type = fstype
+        return best_type
+    except Exception:
+        return ""
+
+
+def run_engine_diagnostics(cfg=None, engine: str = "lite") -> Dict[str, Any]:
+    """Deep runtime self-check for a DuckDB engine.
+
+    Unlike a connection "test", this exercises the runtime to confirm the
+    things that silently break in production:
+
+      * the memory limit is actually applied,
+      * the spill (``temp_directory``) exists, is writable, and is on real
+        disk (not a RAM-backed tmpfs),
+      * a query that exceeds memory genuinely spills to disk instead of OOMing,
+      * the external file cache is in a memory-safe state.
+
+    ``cfg`` is an ``EngineRuntimeConfig`` (or None to use init defaults); the
+    connection is configured exactly like a live Lite/Pro query via
+    ``init_connection`` + ``apply_runtime_pragmas``.  Returns a JSON-serialisable
+    report and never raises.
+    """
+    import shutil
+    import time
+    import uuid
+
+    checks: List[Dict[str, Any]] = []
+
+    def add(cid, label, status, detail="", value=""):
+        checks.append({
+            "id": cid,
+            "label": label,
+            "status": status,
+            "detail": str(detail),
+            "value": "" if value is None else str(value),
+        })
+
+    # 1. Open + configure a connection the same way the engine does.
+    con = None
+    try:
+        con = duckdb.connect()
+        init_connection(con, temp_dir="diagnostics")
+        if cfg is not None:
+            apply_runtime_pragmas(con, cfg)
+        add("connect", "Engine connection", "ok",
+            "Opened and configured a DuckDB connection")
+    except Exception as e:
+        add("connect", "Engine connection", "fail", f"Could not initialise: {e}")
+        return {"engine": engine, "duckdb_version": "", "overall": "fail", "checks": checks}
+
+    # 2. DuckDB version + whether the file cache can be capped on this build.
+    version = ""
+    cappable = False
+    try:
+        version = con.execute("SELECT version()").fetchone()[0]
+        cappable = _external_file_cache_cappable(con)
+        add("version", "DuckDB version", "ok" if cappable else "warn",
+            ("external_file_cache_max_size supported — the file cache can be capped"
+             if cappable else
+             "this build has no external_file_cache_max_size — the file cache "
+             "cannot be capped, so it is disabled to stay memory-safe"),
+            version)
+    except Exception as e:
+        add("version", "DuckDB version", "warn", f"version() failed: {e}")
+
+    # 3. Memory limit effective?
+    try:
+        mem = con.execute("SELECT current_setting('memory_limit')").fetchone()[0]
+        low = str(mem).strip().lower()
+        if not mem or low in ("0 bytes", "0", "-1") or "unlimited" in low:
+            add("memory", "Memory limit", "warn",
+                "No effective memory limit — a heavy query can consume all RAM", mem)
+        else:
+            add("memory", "Memory limit", "ok", "PRAGMA memory_limit is active", mem)
+    except Exception as e:
+        add("memory", "Memory limit", "fail", f"Could not read memory_limit: {e}")
+
+    # 4. Thread count.
+    try:
+        th = con.execute("SELECT current_setting('threads')").fetchone()[0]
+        add("threads", "Worker threads", "ok",
+            "More threads add parallelism but also raise simultaneous memory use", th)
+    except Exception as e:
+        add("threads", "Worker threads", "warn", f"Could not read threads: {e}")
+
+    # 5. Spill (temp) directory: set, exists, writable, on real disk?
+    temp_dir = ""
+    try:
+        temp_dir = con.execute(
+            "SELECT current_setting('temp_directory')"
+        ).fetchone()[0] or ""
+    except Exception:
+        temp_dir = ""
+
+    if not temp_dir:
+        add("temp_dir", "Spill directory", "fail",
+            "temp_directory is empty — DuckDB cannot spill, so heavy queries OOM")
+    else:
+        mtds = ""
+        try:
+            mtds = con.execute(
+                "SELECT current_setting('max_temp_directory_size')"
+            ).fetchone()[0]
+        except Exception:
+            mtds = ""
+
+        writable = False
+        werr = ""
+        try:
+            os.makedirs(temp_dir, exist_ok=True)
+            probe = os.path.join(temp_dir, f".st_spill_probe_{uuid.uuid4().hex}")
+            with open(probe, "wb") as fh:
+                fh.write(b"\0" * (1024 * 1024))  # 1 MiB
+                fh.flush()
+                os.fsync(fh.fileno())
+            os.remove(probe)
+            writable = True
+        except Exception as e:
+            werr = str(e)
+
+        free_gb = None
+        try:
+            free_gb = shutil.disk_usage(temp_dir).free / (1024 ** 3)
+        except Exception:
+            free_gb = None
+        fstype = _filesystem_type(temp_dir)
+        ram_backed = fstype in ("tmpfs", "ramfs")
+
+        parts = [f"path={temp_dir}"]
+        if mtds:
+            parts.append(f"cap={mtds}")
+        if free_gb is not None:
+            parts.append(f"free={free_gb:.1f} GB")
+        if fstype:
+            parts.append(f"fs={fstype}")
+        summary = "; ".join(parts)
+
+        if not writable:
+            add("temp_dir", "Spill directory writable", "fail",
+                f"Cannot write to the spill directory — queries OOM instead of "
+                f"spilling. {werr} ({summary})", temp_dir)
+        elif ram_backed:
+            add("temp_dir", "Spill directory writable", "warn",
+                f"Writable but RAM-backed ({fstype}) — spilling here consumes memory "
+                f"instead of relieving it; mount a real disk volume. ({summary})",
+                temp_dir)
+        elif free_gb is not None and free_gb < 1.0:
+            add("temp_dir", "Spill directory writable", "warn",
+                f"Writable but low free space ({free_gb:.1f} GB) — large spills may "
+                f"fail. ({summary})", temp_dir)
+        else:
+            add("temp_dir", "Spill directory writable", "ok",
+                f"Wrote and removed a 1 MiB probe file. {summary}", temp_dir)
+
+    # 6. Force a real disk spill under memory pressure (end-to-end proof).
+    spill = None
+    try:
+        spill = duckdb.connect()
+        init_connection(spill, temp_dir="diagnostics")
+        spill.execute("PRAGMA memory_limit='256MB';")
+        spill.execute("SET threads=4;")
+        spill.execute("SET preserve_insertion_order=false;")
+        t0 = time.perf_counter()
+        # ~3M rows carrying a wide ~150-byte payload (~465 MB) sorted by a
+        # scrambled key under a 256 MB cap: the working set cannot fit in
+        # memory, so completion proves DuckDB spilled the payload to disk.
+        # The cheap integer sort key keeps it fast (<1 s) while the 256 MB cap
+        # sits far above the pinned-overhead floor, so a healthy disk never
+        # false-fails.
+        n = spill.execute(
+            "SELECT count(*) FROM ("
+            "SELECT hash(i) AS h, repeat('x', 140) || i::VARCHAR AS pad "
+            "FROM range(3000000) t(i) ORDER BY h"
+            ") q"
+        ).fetchone()[0]
+        ms = (time.perf_counter() - t0) * 1000.0
+        add("spill", "Disk spill under pressure", "ok",
+            f"Sorted {n:,} rows (~465 MB) under a 256 MB limit in {ms:.0f} ms — "
+            "DuckDB spilled to disk instead of failing", f"{n:,} rows")
+    except Exception as e:
+        msg = str(e)
+        if "out of memory" in msg.lower() or "failed to pin" in msg.lower():
+            add("spill", "Disk spill under pressure", "fail",
+                "A query that must spill ran out of memory instead — the spill "
+                f"directory is not usable for spilling. {msg}")
+        else:
+            add("spill", "Disk spill under pressure", "warn",
+                f"Spill probe did not complete: {msg}")
+    finally:
+        if spill is not None:
+            try:
+                spill.close()
+            except Exception:
+                pass
+
+    # 7. External file cache memory safety.
+    try:
+        efc = con.execute(
+            "SELECT current_setting('enable_external_file_cache')"
+        ).fetchone()[0]
+        efc_on = str(efc).strip().lower() in ("true", "1")
+        cache_cfg = ""
+        if cfg is not None:
+            cache_cfg = normalize_memory_size(
+                getattr(cfg, "duckdb_external_cache_size", ""), default=""
+            )
+        if efc_on and not cappable:
+            add("cache", "External file cache", "fail",
+                "Cache is ON but this build cannot cap it — it grows to the memory "
+                "limit and causes OOM", "on · uncapped")
+        elif efc_on and cappable:
+            add("cache", "External file cache", "ok",
+                f"Cache is ON and capped at {cache_cfg or 'the configured size'}",
+                "on · capped")
+        else:
+            add("cache", "External file cache", "ok",
+                "Cache is OFF — memory-safe; remote files are re-fetched per query "
+                "(set a Disk cache size on a cap-capable build to speed up repeats)",
+                "off")
+    except Exception as e:
+        add("cache", "External file cache", "warn", f"Could not read cache state: {e}")
+
+    try:
+        con.close()
+    except Exception:
+        pass
+
+    rank = {"ok": 0, "warn": 1, "fail": 2}
+    overall = "ok"
+    for c in checks:
+        if rank.get(c["status"], 0) > rank.get(overall, 0):
+            overall = c["status"]
+
+    return {
+        "engine": engine,
+        "duckdb_version": version,
+        "overall": overall,
+        "checks": checks,
+    }
+
+
 # =========================================================
 # RBAC view creation
 # =========================================================

{supertable-2.2.6 → supertable-2.2.8}/supertable/engine/engine_config.py

@@ -58,14 +58,57 @@ defaults, so behaviour is identical whether a value flows through ``settings``
 from __future__ import annotations
 
 import os
+import re
 from dataclasses import dataclass
 from typing import Any, Dict, Optional, Tuple
 
 
+# Optional decimal number followed by an optional byte unit (decimal KB/MB/GB/TB
+# or binary KiB/MiB/GiB/TiB), case-insensitive, whitespace-tolerant.
+_MEM_SIZE_RE = re.compile(r"^(\d+(?:\.\d+)?)\s*([KMGT]i?B)?$", re.IGNORECASE)
+
+
+def normalize_memory_size(value: Any, default: str = "1GB", *, bare_unit: str = "GB") -> str:
+    """Coerce a memory-size value into a DuckDB-valid, unit-suffixed string.
+
+    The engine UI collects a bare integer (interpreted as ``bare_unit`` — GB),
+    while env vars / older Redis docs may already carry a unit.  DuckDB's
+    ``PRAGMA memory_limit`` parser rejects a unit-less number
+    (``Unknown unit for memory: ''``), so this guarantees a value the parser
+    accepts before it is ever sent:
+
+      * bare number            -> ``"<n>GB"``       ("2"    -> "2GB")
+      * number + unit          -> canonical casing  ("2gib" -> "2GiB")
+      * empty / None           -> ``default``       (""/None -> default)
+      * non-positive / garbage -> ``default``       ("0", "x" -> default)
+
+    ``default`` is returned verbatim, so pass ``""`` when "unset" is the
+    desired fallback (e.g. the external file cache, where empty == disabled).
+    """
+    s = ("" if value is None else str(value)).strip()
+    if not s:
+        return default
+    m = _MEM_SIZE_RE.match(s)
+    if not m:
+        return default
+    num, unit = m.group(1), m.group(2)
+    try:
+        if float(num) <= 0:
+            return default
+    except ValueError:
+        return default
+    if unit:
+        u = unit.upper()
+        unit = (u[0] + "iB") if u.endswith("IB") else u
+    else:
+        unit = bare_unit
+    return f"{num}{unit}"
+
+
 # Shared auto-pick thresholds: key → (env var, built-in default as string).
 _SHARED_SPEC: Dict[str, Tuple[str, str]] = {
     "engine_lite_max_bytes":  ("SUPERTABLE_ENGINE_LITE_MAX_BYTES",  str(100 * 1024 * 1024)),
-    "engine_spark_min_bytes": ("SUPERTABLE_ENGINE_SPARK_MIN_BYTES", str(10 * 1024 * 1024 * 1024)),
+    "engine_spark_min_bytes": ("SUPERTABLE_ENGINE_SPARK_MIN_BYTES", str(0)),
     "engine_freshness_sec":   ("SUPERTABLE_ENGINE_FRESHNESS_SEC",   "300"),
 }
 
@@ -166,13 +209,13 @@ def _build_runtime(redis_cfg: Dict[str, Any], engine: str) -> EngineRuntimeConfi
 
     return EngineRuntimeConfig(
         engine_lite_max_bytes=_to_int(sv["engine_lite_max_bytes"], 100 * 1024 * 1024),
-        engine_spark_min_bytes=_to_int(sv["engine_spark_min_bytes"], 10 * 1024 * 1024 * 1024),
+        engine_spark_min_bytes=_to_int(sv["engine_spark_min_bytes"], 0),
         engine_freshness_sec=_to_int(sv["engine_freshness_sec"], 300),
-        duckdb_memory_limit=dv["duckdb_memory_limit"].strip() or "1GB",
+        duckdb_memory_limit=normalize_memory_size(dv["duckdb_memory_limit"], default="1GB"),
         duckdb_io_multiplier=_to_float(dv["duckdb_io_multiplier"], 3.0),
         duckdb_threads=(threads if threads > 0 else None),
         duckdb_http_timeout=(http if http > 0 else None),
-        duckdb_external_cache_size=dv["duckdb_external_cache_size"].strip(),
+        duckdb_external_cache_size=normalize_memory_size(dv["duckdb_external_cache_size"], default=""),
     )