supertable 2.2.6__tar.gz → 2.2.7__tar.gz

{supertable-2.2.6/supertable.egg-info → supertable-2.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supertable
-Version: 2.2.6
+Version: 2.2.7
 Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
 Author: Levente Kupas
 Author-email: Levente Kupas <lkupas@kladnasoft.com>

{supertable-2.2.6 → supertable-2.2.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "supertable"
-version = "2.2.6"
+version = "2.2.7"
 description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
 readme = "README.md"
 requires-python = ">=3.10"

{supertable-2.2.6 → supertable-2.2.7}/setup.py

@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
 
 setup(
     name="supertable",
-    version="2.2.6",
+    version="2.2.7",
     description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
     long_description=long_description,
     long_description_content_type="text/markdown",

{supertable-2.2.6 → supertable-2.2.7}/supertable/__init__.py

@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
 project documentation for the full API surface.
 """
 
-__version__ = "2.2.6"
+__version__ = "2.2.7"
 
 # Re-export the core public surface so users can do ``from supertable import …``
 # instead of remembering submodule paths.

{supertable-2.2.6 → supertable-2.2.7}/supertable/data_reader.py

@@ -44,10 +44,20 @@ class DataReader:
       - Execution to Executor (DuckDB/Spark)
     """
 
-    def __init__(self, super_name: str, organization: str, query: str):
+    def __init__(
+        self,
+        super_name: str,
+        organization: str,
+        query: str,
+        source: str = "sdk",
+    ):
         self.super_name = super_name
         self.organization = organization
         self.query = query
+        # Query origin surfaced in the reads monitoring tab. "sdk" is the
+        # default for direct SDK callers; the API/OData/MCP entry points
+        # pass "api"/"odata"/"mcp" so each query records where it came from.
+        self.source = source
 
         self.storage: StorageInterface = get_storage()
 
@@ -164,6 +174,9 @@ class DataReader:
                 current_meta_path="redis://meta/root",
                 query=parser.original_query,
             )
+            # Stamp the call origin so plan_extender records it on the read
+            # monitoring entry (defaults to "api" downstream if unset).
+            self.query_plan_manager.source_type = self.source
             self._log_ctx = f"[qid={self.query_plan_manager.query_id} qh={self.query_plan_manager.query_hash}] "
             self.query_plan_manager.original_table = ", ".join(t.simple_name for t in physical_tables) if physical_tables else ""
 
@@ -311,16 +324,25 @@ def query_sql(
         limit: int,
         engine: Any,
         role_name: str,
+        source: str = "sdk",
+        out: Optional[Dict[str, Any]] = None,
 ) -> Tuple[List[str], List[List[Any]], List[Dict[str, Any]]]:
     """
     Execute SQL query and return results in the format expected by MCP server.
     Returns: (columns, rows, columns_meta)
+
+    ``source`` tags the query origin on the read monitoring entry
+    (defaults to "sdk"; the MCP server passes "mcp"). When an ``out``
+    dict is supplied it is populated with ``query_id``/``query_hash`` so
+    the caller can correlate its own audit log to this read record.
     """
     # Safety guard: ensure a LIMIT is present so unbounded queries don't
     # overwhelm the MCP response payload.
     sql = _ensure_sql_limit(sql, default_limit=limit)
 
-    reader = DataReader(organization=organization, super_name=super_name, query=sql)
+    reader = DataReader(
+        organization=organization, super_name=super_name, query=sql, source=source,
+    )
 
     # Execute the query
     result_df, status, message = reader.execute(
@@ -329,6 +351,15 @@ def query_sql(
         with_scan=False,
     )
 
+    # Expose the query identity so the caller (e.g. the MCP audit log) can
+    # link back to this read's monitoring entry. Populated even on error,
+    # since the QueryPlanManager is created before execution.
+    if out is not None:
+        qpm = reader.query_plan_manager
+        if qpm is not None:
+            out["query_id"] = qpm.query_id
+            out["query_hash"] = qpm.query_hash
+
     if status == Status.ERROR:
         raise RuntimeError(f"Query execution failed: {message}")
 

{supertable-2.2.6 → supertable-2.2.7}/supertable/engine/engine_common.py

@@ -14,6 +14,7 @@ from sqlglot import exp
 from supertable.config.defaults import logger
 from supertable.config.settings import settings
 from supertable.config.homedir import get_app_home
+from supertable.engine.engine_config import normalize_memory_size
 
 
 # =========================================================
@@ -578,8 +579,18 @@ def init_connection(
     # Resolve memory limit.
     # Single env var SUPERTABLE_DUCKDB_MEMORY_LIMIT controls both executors.
     # The `memory_limit` argument is the caller's fallback when the env var is absent.
-    effective_memory_limit = settings.SUPERTABLE_DUCKDB_MEMORY_LIMIT or memory_limit
-    con.execute(f"PRAGMA memory_limit='{effective_memory_limit}';")
+    # normalize_memory_size guarantees a unit-suffixed value so DuckDB's parser
+    # never rejects a bare number (e.g. a UI-supplied "2" -> "2GB").
+    effective_memory_limit = normalize_memory_size(
+        settings.SUPERTABLE_DUCKDB_MEMORY_LIMIT or memory_limit, default="1GB"
+    )
+    try:
+        con.execute(f"PRAGMA memory_limit='{effective_memory_limit}';")
+    except Exception as e:
+        logger.warning(
+            f"[duckdb.init] memory_limit='{effective_memory_limit}' rejected: {e}; "
+            f"keeping DuckDB default"
+        )
 
     # Absolute temp path is required for DuckDB to actually spill to disk.
     # Prefer a path rooted under the app home (~/supertable), which is
@@ -647,11 +658,14 @@ def apply_runtime_pragmas(con: duckdb.DuckDBPyConnection, cfg) -> None:
     if cfg is None:
         return
 
-    memory_limit = (cfg.duckdb_memory_limit or "").strip() or "1GB"
+    # normalize_memory_size guarantees a unit (UI sends a bare "2" -> "2GB");
+    # a rejected value is logged rather than silently swallowed so a bad
+    # config can never quietly leave the connection at the wrong limit.
+    memory_limit = normalize_memory_size(cfg.duckdb_memory_limit, default="1GB")
     try:
         con.execute(f"PRAGMA memory_limit='{memory_limit}';")
-    except Exception:
-        pass
+    except Exception as e:
+        logger.warning(f"[duckdb.pragma] memory_limit='{memory_limit}' rejected: {e}")
 
     # Explicit thread count wins; otherwise derive from the live memory limit
     # and IO multiplier (same formula as init_connection).
@@ -671,13 +685,32 @@ def apply_runtime_pragmas(con: duckdb.DuckDBPyConnection, cfg) -> None:
         except Exception:
             pass
 
-    cache_size = (cfg.duckdb_external_cache_size or "").strip()
+    cache_size = normalize_memory_size(cfg.duckdb_external_cache_size, default="")
     if cache_size:
         try:
             con.execute("SET enable_external_file_cache=true;")
-            con.execute(f"SET external_file_cache_max_size='{sanitize_sql_string(cache_size)}';")
-        except Exception:
-            pass
+        except Exception as e:
+            logger.warning(f"[duckdb.pragma] enable_external_file_cache failed: {e}")
+        # The size cap is version-dependent: ``external_file_cache_max_size``
+        # does not exist on every DuckDB build (e.g. 1.5.x).  Probe the
+        # settings catalog and only set it when present so we neither raise
+        # nor silently swallow a genuinely unsupported setting.
+        try:
+            has_cap = bool(con.execute(
+                "SELECT 1 FROM duckdb_settings() "
+                "WHERE name='external_file_cache_max_size'"
+            ).fetchone())
+            if has_cap:
+                con.execute(
+                    f"SET external_file_cache_max_size='{sanitize_sql_string(cache_size)}';"
+                )
+            else:
+                logger.debug(
+                    "[duckdb.pragma] external_file_cache_max_size unsupported on this "
+                    "DuckDB version; cache enabled without a size cap"
+                )
+        except Exception as e:
+            logger.warning(f"[duckdb.pragma] external_file_cache_max_size failed: {e}")
 
 
 # =========================================================

{supertable-2.2.6 → supertable-2.2.7}/supertable/engine/engine_config.py

@@ -58,10 +58,53 @@ defaults, so behaviour is identical whether a value flows through ``settings``
 from __future__ import annotations
 
 import os
+import re
 from dataclasses import dataclass
 from typing import Any, Dict, Optional, Tuple
 
 
+# Optional decimal number followed by an optional byte unit (decimal KB/MB/GB/TB
+# or binary KiB/MiB/GiB/TiB), case-insensitive, whitespace-tolerant.
+_MEM_SIZE_RE = re.compile(r"^(\d+(?:\.\d+)?)\s*([KMGT]i?B)?$", re.IGNORECASE)
+
+
+def normalize_memory_size(value: Any, default: str = "1GB", *, bare_unit: str = "GB") -> str:
+    """Coerce a memory-size value into a DuckDB-valid, unit-suffixed string.
+
+    The engine UI collects a bare integer (interpreted as ``bare_unit`` — GB),
+    while env vars / older Redis docs may already carry a unit.  DuckDB's
+    ``PRAGMA memory_limit`` parser rejects a unit-less number
+    (``Unknown unit for memory: ''``), so this guarantees a value the parser
+    accepts before it is ever sent:
+
+      * bare number            -> ``"<n>GB"``       ("2"    -> "2GB")
+      * number + unit          -> canonical casing  ("2gib" -> "2GiB")
+      * empty / None           -> ``default``       (""/None -> default)
+      * non-positive / garbage -> ``default``       ("0", "x" -> default)
+
+    ``default`` is returned verbatim, so pass ``""`` when "unset" is the
+    desired fallback (e.g. the external file cache, where empty == disabled).
+    """
+    s = ("" if value is None else str(value)).strip()
+    if not s:
+        return default
+    m = _MEM_SIZE_RE.match(s)
+    if not m:
+        return default
+    num, unit = m.group(1), m.group(2)
+    try:
+        if float(num) <= 0:
+            return default
+    except ValueError:
+        return default
+    if unit:
+        u = unit.upper()
+        unit = (u[0] + "iB") if u.endswith("IB") else u
+    else:
+        unit = bare_unit
+    return f"{num}{unit}"
+
+
 # Shared auto-pick thresholds: key → (env var, built-in default as string).
 _SHARED_SPEC: Dict[str, Tuple[str, str]] = {
     "engine_lite_max_bytes":  ("SUPERTABLE_ENGINE_LITE_MAX_BYTES",  str(100 * 1024 * 1024)),
@@ -168,11 +211,11 @@ def _build_runtime(redis_cfg: Dict[str, Any], engine: str) -> EngineRuntimeConfi
         engine_lite_max_bytes=_to_int(sv["engine_lite_max_bytes"], 100 * 1024 * 1024),
         engine_spark_min_bytes=_to_int(sv["engine_spark_min_bytes"], 10 * 1024 * 1024 * 1024),
         engine_freshness_sec=_to_int(sv["engine_freshness_sec"], 300),
-        duckdb_memory_limit=dv["duckdb_memory_limit"].strip() or "1GB",
+        duckdb_memory_limit=normalize_memory_size(dv["duckdb_memory_limit"], default="1GB"),
         duckdb_io_multiplier=_to_float(dv["duckdb_io_multiplier"], 3.0),
         duckdb_threads=(threads if threads > 0 else None),
         duckdb_http_timeout=(http if http > 0 else None),
-        duckdb_external_cache_size=dv["duckdb_external_cache_size"].strip(),
+        duckdb_external_cache_size=normalize_memory_size(dv["duckdb_external_cache_size"], default=""),
     )
 
 

{supertable-2.2.6 → supertable-2.2.7}/supertable/engine/executor.py

@@ -62,6 +62,47 @@ class Executor:
                 self._catalog = False  # sentinel: construction failed, do not retry
         return self._catalog or None
 
+    def _spark_min_bytes(self, cfg: EngineRuntimeConfig) -> int:
+        """Effective lower byte bound for routing a query to Spark under AUTO.
+
+        Couples the configured policy floor (``engine_spark_min_bytes``) with the
+        registered Spark fleet so the boundary is::
+
+            max(engine_spark_min_bytes, min(active cluster min_bytes))
+
+        Rationale:
+          * The ``max`` keeps the admin's global floor effective — keep
+            ``engine_spark_min_bytes`` high to hold medium jobs on DuckDB even
+            when a cluster would accept them (set it to 0 to let the fleet alone
+            drive routing).
+          * Folding in the fleet minimum means AUTO never routes a job *below*
+            what any active cluster accepts — which would otherwise make
+            ``select_spark_cluster`` return nothing and hard-fail the query
+            (see spark_thrift.py ``_select_cluster``).
+
+        Degrades to the policy floor alone when no catalog is reachable or no
+        active clusters are registered.
+        """
+        policy = cfg.engine_spark_min_bytes
+        catalog = self._get_catalog()
+        if catalog is None:
+            return policy
+        try:
+            clusters = catalog.list_spark_clusters(self.organization) or []
+        except Exception:
+            return policy
+        mins = []
+        for c in clusters:
+            if not isinstance(c, dict) or c.get("status") != "active":
+                continue
+            try:
+                mins.append(int(c.get("min_bytes", 0)))
+            except (TypeError, ValueError):
+                continue
+        if not mins:
+            return policy
+        return max(policy, min(mins))
+
     def _auto_pick(self, reflection: Reflection, cfg: EngineRuntimeConfig) -> Engine:
         """Select the best engine based on data size and freshness.
 
@@ -86,12 +127,17 @@ class Executor:
           SUPERTABLE_ENGINE_LITE_MAX_BYTES   – upper bound for Lite (default 100 MB)
           SUPERTABLE_ENGINE_SPARK_MIN_BYTES  – lower bound for Spark (default 10 GB)
           SUPERTABLE_ENGINE_FRESHNESS_SEC    – age threshold in seconds (default 300)
+
+        The Spark lower bound is additionally coupled to the registered fleet
+        (see :meth:`_spark_min_bytes`): the effective boundary is the larger of
+        the configured floor and the smallest ``min_bytes`` across active
+        clusters, so AUTO only routes to Spark when some cluster will take it.
         """
         bytes_total = reflection.reflection_bytes
 
         # --- thresholds (resolved live from org system config) ---
         lite_max = cfg.engine_lite_max_bytes
-        spark_min = cfg.engine_spark_min_bytes
+        spark_min = self._spark_min_bytes(cfg)
         freshness_threshold_s = cfg.engine_freshness_sec
 
         # --- freshness: how long ago was the most recent snapshot updated ---

supertable-2.2.7/supertable/engine/tests/test_engine_config.py

@@ -0,0 +1,81 @@
+# supertable/engine/tests/test_engine_config.py
+"""Tests for supertable.engine.engine_config.
+
+Focus: ``normalize_memory_size`` (the unit "extender") and that the resolver
+emits unit-suffixed, DuckDB-parseable memory values regardless of how the UI /
+Redis stored them.  The core guarantee is that a bare number can never reach
+``PRAGMA memory_limit`` and raise ``Unknown unit for memory: ''``.
+"""
+
+from __future__ import annotations
+
+import duckdb
+import pytest
+
+from supertable.engine.engine_config import (
+    normalize_memory_size,
+    resolve_engine_configs,
+)
+
+
+class TestNormalizeMemorySize:
+    @pytest.mark.parametrize(
+        "value,expected",
+        [
+            ("2", "2GB"),          # bare number -> GB
+            ("8", "8GB"),
+            ("50", "50GB"),
+            (2, "2GB"),            # non-string input
+            ("2.5", "2.5GB"),
+            ("2GB", "2GB"),        # already valid -> passthrough
+            ("2gb", "2GB"),        # lowercase -> canonical
+            ("2gib", "2GiB"),      # binary unit canonicalised
+            ("512mb", "512MB"),
+            ("  4 GB ", "4GB"),    # whitespace tolerant
+        ],
+    )
+    def test_extends_and_canonicalises(self, value, expected):
+        assert normalize_memory_size(value) == expected
+
+    @pytest.mark.parametrize("value", ["", "   ", None, "0", "-5", "abc", "GB", "1XB"])
+    def test_invalid_falls_back_to_default(self, value):
+        assert normalize_memory_size(value, default="1GB") == "1GB"
+
+    def test_empty_default_for_cache(self):
+        # Cache uses ""(=disabled) as the fallback rather than a size.
+        assert normalize_memory_size("", default="") == ""
+        assert normalize_memory_size("0", default="") == ""
+        assert normalize_memory_size("50", default="") == "50GB"
+
+    @pytest.mark.parametrize(
+        "raw", ["2", "8", "50", "0", "", None, "abc", "2gib", "512mb", "-1"]
+    )
+    def test_output_always_accepted_by_duckdb(self, raw):
+        """The whole point: the normalized value never raises a ParserException."""
+        con = duckdb.connect(":memory:")
+        normalized = normalize_memory_size(raw, default="1GB")
+        con.execute(f"PRAGMA memory_limit='{normalized}';")  # must not raise
+        con.close()
+
+
+class TestResolverNormalizes:
+    def test_bare_numbers_become_unit_suffixed(self):
+        # Simulates the exact Redis doc the user reported (bare 2 / 8 / 50).
+        stored = {
+            "lite": {"duckdb_memory_limit": "2", "duckdb_external_cache_size": "50"},
+            "pro": {"duckdb_memory_limit": "8", "duckdb_external_cache_size": "50"},
+        }
+
+        class _Catalog:
+            def get_engine_config(self, org):
+                return stored
+
+        cfgs = resolve_engine_configs("kladna-soft", _Catalog())
+        assert cfgs["lite"].duckdb_memory_limit == "2GB"
+        assert cfgs["pro"].duckdb_memory_limit == "8GB"
+        assert cfgs["lite"].duckdb_external_cache_size == "50GB"
+
+        # And the resolved values are all DuckDB-parseable.
+        con = duckdb.connect(":memory:")
+        con.execute(f"PRAGMA memory_limit='{cfgs['pro'].duckdb_memory_limit}';")
+        con.close()

supertable-2.2.7/supertable/engine/tests/test_engine_routing.py

@@ -0,0 +1,110 @@
+# supertable/engine/tests/test_engine_routing.py
+"""Tests for the fleet-aware Spark routing boundary.
+
+``Executor._spark_min_bytes`` decides the byte size at which AUTO hands a query
+to Spark.  It couples the configured policy floor with the registered Thrift
+fleet so the boundary is ``max(engine_spark_min_bytes, min(active min_bytes))``:
+
+  * the policy floor is never undercut (medium jobs can be held on DuckDB), and
+  * AUTO never routes a job below what any active cluster accepts (which would
+    make ``select_spark_cluster`` return nothing and hard-fail the query).
+"""
+
+from __future__ import annotations
+
+from supertable.engine.executor import Executor
+from supertable.engine.engine_config import EngineRuntimeConfig
+
+GIB = 1024 ** 3
+
+
+def _cfg(spark_min_bytes: int) -> EngineRuntimeConfig:
+    return EngineRuntimeConfig(
+        engine_lite_max_bytes=100 * 1024 * 1024,
+        engine_spark_min_bytes=spark_min_bytes,
+        engine_freshness_sec=300,
+        duckdb_memory_limit="1GB",
+        duckdb_io_multiplier=3.0,
+        duckdb_threads=None,
+        duckdb_http_timeout=None,
+        duckdb_external_cache_size="",
+    )
+
+
+class _Catalog:
+    def __init__(self, clusters):
+        self._clusters = clusters
+
+    def list_spark_clusters(self, org):
+        return self._clusters
+
+
+class _RaisingCatalog:
+    def list_spark_clusters(self, org):
+        raise RuntimeError("redis down")
+
+
+def _executor(catalog) -> Executor:
+    # Bypass __init__ (which builds a DuckDBLite); we only exercise the boundary.
+    e = Executor.__new__(Executor)
+    e._catalog = catalog
+    e.organization = "kladna-soft"
+    return e
+
+
+def _active(min_bytes, max_bytes=0):
+    return {"status": "active", "min_bytes": min_bytes, "max_bytes": max_bytes}
+
+
+def test_no_catalog_uses_policy_floor():
+    assert _executor(None)._spark_min_bytes(_cfg(10 * GIB)) == 10 * GIB
+
+
+def test_empty_fleet_uses_policy_floor():
+    assert _executor(_Catalog([]))._spark_min_bytes(_cfg(10 * GIB)) == 10 * GIB
+
+
+def test_catalog_error_falls_back_to_policy():
+    assert _executor(_RaisingCatalog())._spark_min_bytes(_cfg(10 * GIB)) == 10 * GIB
+
+
+def test_fleet_min_below_policy_keeps_policy():
+    # Cluster accepts >=1 GiB, but the admin floor is 10 GiB: floor wins.
+    cat = _Catalog([_active(1 * GIB)])
+    assert _executor(cat)._spark_min_bytes(_cfg(10 * GIB)) == 10 * GIB
+
+
+def test_fleet_min_above_policy_raises_boundary():
+    # No cluster will take a job under 20 GiB, so AUTO must not route below it.
+    cat = _Catalog([_active(20 * GIB)])
+    assert _executor(cat)._spark_min_bytes(_cfg(10 * GIB)) == 20 * GIB
+
+
+def test_smallest_active_cluster_drives_the_fleet_min():
+    cat = _Catalog([_active(50 * GIB), _active(20 * GIB), _active(30 * GIB)])
+    # min active = 20 GiB; policy 10 GiB -> 20 GiB.
+    assert _executor(cat)._spark_min_bytes(_cfg(10 * GIB)) == 20 * GIB
+
+
+def test_inactive_clusters_are_ignored():
+    cat = _Catalog([
+        {"status": "draining", "min_bytes": 1 * GIB, "max_bytes": 0},
+        {"status": "offline", "min_bytes": 2 * GIB, "max_bytes": 0},
+    ])
+    # No active cluster -> policy floor.
+    assert _executor(cat)._spark_min_bytes(_cfg(10 * GIB)) == 10 * GIB
+
+
+def test_zero_policy_lets_fleet_drive():
+    # engine_spark_min_bytes=0 -> pure fleet-min behaviour.
+    cat = _Catalog([_active(1 * GIB), _active(4 * GIB)])
+    assert _executor(cat)._spark_min_bytes(_cfg(0)) == 1 * GIB
+
+
+def test_malformed_min_bytes_is_skipped_not_fatal():
+    cat = _Catalog([
+        {"status": "active", "min_bytes": "not-a-number", "max_bytes": 0},
+        _active(7 * GIB),
+    ])
+    # Bad row skipped; good row drives the fleet min. policy 1 GiB -> 7 GiB.
+    assert _executor(cat)._spark_min_bytes(_cfg(1 * GIB)) == 7 * GIB