supertable 2.2.8__tar.gz → 2.3.0__tar.gz

{supertable-2.2.8/supertable.egg-info → supertable-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supertable
-Version: 2.2.8
+Version: 2.3.0
 Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
 Author: Levente Kupas
 Author-email: Levente Kupas <lkupas@kladnasoft.com>

{supertable-2.2.8 → supertable-2.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "supertable"
-version = "2.2.8"
+version = "2.3.0"
 description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
 readme = "README.md"
 requires-python = ">=3.10"

{supertable-2.2.8 → supertable-2.3.0}/setup.py

@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
 
 setup(
     name="supertable",
-    version="2.2.8",
+    version="2.3.0",
     description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
     long_description=long_description,
     long_description_content_type="text/markdown",

{supertable-2.2.8 → supertable-2.3.0}/supertable/__init__.py

@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
 project documentation for the full API surface.
 """
 
-__version__ = "2.2.8"
+__version__ = "2.3.0"
 
 # Re-export the core public surface so users can do ``from supertable import …``
 # instead of remembering submodule paths.

{supertable-2.2.8 → supertable-2.3.0}/supertable/config/defaults.py

@@ -23,6 +23,7 @@ _VALID_LOG_LEVELS = frozenset({"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"})
 class Default:
     MAX_MEMORY_CHUNK_SIZE: int = 16 * 1024 * 1024
     MAX_OVERLAPPING_FILES: int = 100
+    MAX_TOMBSTONE_ROWS: int = 1_000_000
     DEFAULT_TIMEOUT_SEC: int = 60
     DEFAULT_LOCK_DURATION_SEC: int = 30
     LOG_LEVEL: str = "INFO"
@@ -51,6 +52,7 @@ def load_defaults_from_env(env_file: str | None = None, prefer_system: bool = Tr
     return Default(
         MAX_MEMORY_CHUNK_SIZE=settings.MAX_MEMORY_CHUNK_SIZE,
         MAX_OVERLAPPING_FILES=settings.MAX_OVERLAPPING_FILES,
+        MAX_TOMBSTONE_ROWS=settings.MAX_TOMBSTONE_ROWS,
         DEFAULT_TIMEOUT_SEC=settings.DEFAULT_TIMEOUT_SEC,
         DEFAULT_LOCK_DURATION_SEC=settings.DEFAULT_LOCK_DURATION_SEC,
         LOG_LEVEL=log_level,

{supertable-2.2.8 → supertable-2.3.0}/supertable/config/settings.py

@@ -116,6 +116,7 @@ class Settings:
     # ── Defaults (legacy config/defaults.py) ─────────────────────────
     MAX_MEMORY_CHUNK_SIZE: int = 16 * 1024 * 1024  # MAX_MEMORY_CHUNK_SIZE
     MAX_OVERLAPPING_FILES: int = 100               # MAX_OVERLAPPING_FILES
+    MAX_TOMBSTONE_ROWS: int = 1_000_000            # MAX_TOMBSTONE_ROWS
     DEFAULT_TIMEOUT_SEC: int = 60                  # DEFAULT_TIMEOUT_SEC
     DEFAULT_LOCK_DURATION_SEC: int = 30            # DEFAULT_LOCK_DURATION_SEC
     IS_SHOW_TIMING: bool = False                   # IS_SHOW_TIMING
@@ -259,6 +260,16 @@ class Settings:
     # ── Meta Reader / Caching ────────────────────────────────────────
     SUPERTABLE_SUPER_META_CACHE_TTL_S: Optional[float] = None  # SUPERTABLE_SUPER_META_CACHE_TTL_S
 
+    # Max number of tables whose *latest* stats artifact is held in the
+    # in-process stats cache (one DataFrame per table; older versions are
+    # always read fresh and never cached).  0 disables caching.
+    SUPERTABLE_STATS_CACHE_MAX_TABLES: int = 64   # SUPERTABLE_STATS_CACHE_MAX_TABLES
+
+    # Read-path file pruning: when True the estimator uses the stats artifact to
+    # drop parquet files that provably can't satisfy a query's WHERE predicates.
+    # Conservative (never drops a file that could match); set False to disable.
+    SUPERTABLE_READ_PRUNING_ENABLED: bool = True  # SUPERTABLE_READ_PRUNING_ENABLED
+
     # ── Audit ────────────────────────────────────────────────────────
     # Audit is OFF by default.  Enable per-organization in the WebUI
     # /ui/audit → Compliance tab (persisted at supertable:{org}:system:audit:config),
@@ -366,6 +377,7 @@ def _build_settings() -> Settings:
         # ── Defaults ─────────────────────────────────────────────────
         MAX_MEMORY_CHUNK_SIZE=_env_int("MAX_MEMORY_CHUNK_SIZE", 16 * 1024 * 1024),
         MAX_OVERLAPPING_FILES=_env_int("MAX_OVERLAPPING_FILES", 100),
+        MAX_TOMBSTONE_ROWS=_env_int("MAX_TOMBSTONE_ROWS", 1_000_000),
         DEFAULT_TIMEOUT_SEC=_env_int("DEFAULT_TIMEOUT_SEC", 60),
         DEFAULT_LOCK_DURATION_SEC=_env_int("DEFAULT_LOCK_DURATION_SEC", 30),
         IS_SHOW_TIMING=_env_bool("IS_SHOW_TIMING", True),
@@ -509,6 +521,8 @@ def _build_settings() -> Settings:
 
         # ── Meta Reader / Caching ────────────────────────────────────
         SUPERTABLE_SUPER_META_CACHE_TTL_S=meta_ttl,
+        SUPERTABLE_STATS_CACHE_MAX_TABLES=_env_int("SUPERTABLE_STATS_CACHE_MAX_TABLES", 64),
+        SUPERTABLE_READ_PRUNING_ENABLED=_env_bool("SUPERTABLE_READ_PRUNING_ENABLED", True),
 
         # ── Audit ────────────────────────────────────────────────────
         SUPERTABLE_AUDIT_ENABLED=_env_bool("SUPERTABLE_AUDIT_ENABLED", False),

{supertable-2.2.8 → supertable-2.3.0}/supertable/config/tests/test_defaults.py

@@ -78,6 +78,7 @@ class TestLoadDefaultsFromEnv:
             SUPERTABLE_LOG_LEVEL="WARNING",
             MAX_MEMORY_CHUNK_SIZE=12345,
             MAX_OVERLAPPING_FILES=7,
+            MAX_TOMBSTONE_ROWS=999,
             DEFAULT_TIMEOUT_SEC=11,
             DEFAULT_LOCK_DURATION_SEC=22,
             IS_SHOW_TIMING=False,
@@ -89,6 +90,7 @@ class TestLoadDefaultsFromEnv:
         assert d.LOG_LEVEL == "WARNING"
         assert d.MAX_MEMORY_CHUNK_SIZE == 12345
         assert d.MAX_OVERLAPPING_FILES == 7
+        assert d.MAX_TOMBSTONE_ROWS == 999
         assert d.DEFAULT_TIMEOUT_SEC == 11
         assert d.DEFAULT_LOCK_DURATION_SEC == 22
         assert d.IS_SHOW_TIMING is False
@@ -101,6 +103,7 @@ class TestLoadDefaultsFromEnv:
             SUPERTABLE_LOG_LEVEL="LOUDEST",
             MAX_MEMORY_CHUNK_SIZE=1,
             MAX_OVERLAPPING_FILES=1,
+            MAX_TOMBSTONE_ROWS=1,
             DEFAULT_TIMEOUT_SEC=1,
             DEFAULT_LOCK_DURATION_SEC=1,
             IS_SHOW_TIMING=False,
@@ -121,6 +124,7 @@ class TestLoadDefaultsFromEnv:
             SUPERTABLE_LOG_LEVEL=level,
             MAX_MEMORY_CHUNK_SIZE=1,
             MAX_OVERLAPPING_FILES=1,
+            MAX_TOMBSTONE_ROWS=1,
             DEFAULT_TIMEOUT_SEC=1,
             DEFAULT_LOCK_DURATION_SEC=1,
             IS_SHOW_TIMING=False,

{supertable-2.2.8 → supertable-2.3.0}/supertable/data_classes.py

@@ -1,5 +1,23 @@
 from dataclasses import dataclass, field
-from typing import Dict, List, Optional, Set
+from typing import Dict, List, NamedTuple, Optional, Set
+
+
+class PredInterval(NamedTuple):
+    """A closed/open range of values a column is *allowed* to take, derived from
+    a SQL WHERE predicate, used for read-path file pruning.
+
+    ``lane`` is one of ``"numeric"`` / ``"string"`` / ``"timestamp"`` and selects
+    which stored stats lane the interval is compared against (numeric unifies the
+    stored ``bigint``/``double`` lanes).  ``lo``/``hi`` are the bounds with
+    ``None`` meaning unbounded (-inf / +inf); ``lo_incl``/``hi_incl`` mark
+    whether each bound is inclusive.  Examples: ``col = 5`` →
+    ``(lane, 5, True, 5, True)``; ``col > 5`` → ``(lane, 5, False, None, True)``.
+    """
+    lane: str
+    lo: object
+    lo_incl: bool
+    hi: object
+    hi_incl: bool
 
 
 @dataclass
@@ -32,39 +50,20 @@ class RbacViewDef:
     where_clause: str = ""
 
 
-@dataclass
-class DedupViewDef:
-    """Dedup-on-read view definition for a single table alias.
-
-    Produced by DataReader from table config in Redis, consumed by executors
-    to create a ROW_NUMBER window view that keeps only the latest row per
-    primary key combination.
-
-    - primary_keys: columns that form the composite key for dedup.
-    - order_column: column to ORDER BY DESC (default ``__timestamp__``).
-    - visible_columns: columns to expose in the dedup view.  When empty or
-      ["*"], all columns except ``__rn__`` are exposed.  When set, only
-      these columns are projected (hiding PK / order cols that the user
-      did not request).
-    """
-    primary_keys: List[str] = field(default_factory=list)
-    order_column: str = "__timestamp__"
-    visible_columns: List[str] = field(default_factory=list)
-
-
 @dataclass
 class TombstoneDef:
-    """Tombstone definition for a single table alias.
+    """Tombstone (deletion-vector) definition for a single table alias.
 
-    Produced by DataReader from the snapshot's ``tombstones`` block,
-    consumed by executors to create a filtering view that excludes
-    soft-deleted keys.
+    Produced by DataReader from the snapshot's ``tombstone`` pointer,
+    consumed by executors to create a view that anti-joins the data on
+    ``__rowid__`` against the deletion-vector parquet, hiding rows that
+    have been logically deleted but not yet physically compacted.
 
-    - primary_keys: columns that form the composite key.
-    - deleted_keys: list of key tuples (each a list of scalars).
+    - tombstone_path: storage path of the deletion-vector parquet
+      (columns ``__file__`` + ``__rowid__``).  ``None`` means no
+      tombstone exists, so no anti-join is applied.
     """
-    primary_keys: List[str] = field(default_factory=list)
-    deleted_keys: List = field(default_factory=list)
+    tombstone_path: Optional[str] = None
 
 
 @dataclass
@@ -79,7 +78,5 @@ class Reflection:
     freshness_ms: int = 0
     # alias -> RbacViewDef.  Empty dict means no RBAC filtering.
     rbac_views: Dict[str, RbacViewDef] = field(default_factory=dict)
-    # alias -> DedupViewDef.  Empty dict means no dedup-on-read.
-    dedup_views: Dict[str, DedupViewDef] = field(default_factory=dict)
     # alias -> TombstoneDef.  Empty dict means no tombstone filtering.
     tombstone_views: Dict[str, TombstoneDef] = field(default_factory=dict)

{supertable-2.2.8 → supertable-2.3.0}/supertable/data_reader.py

@@ -23,8 +23,9 @@ from supertable.rbac.access_control import restrict_read_access  # noqa: F401
 from supertable.engine.data_estimator import DataEstimator
 from supertable.engine.executor import Executor
 from supertable.engine.engine_enum import Engine as engine
-from supertable.data_classes import DedupViewDef, TombstoneDef, RbacViewDef
+from supertable.data_classes import TombstoneDef, RbacViewDef
 from supertable.redis_catalog import RedisCatalog
+from supertable.system_query import classify_query, CommandKind
 
 
 class Status(Enum):
@@ -109,6 +110,82 @@ class DataReader:
                     self.organization, super_name, simple_name
                 )
 
+    def _resolve_latest_stats_file(
+        self, super_name: str, simple_name: str,
+    ) -> Optional[str]:
+        """Return the ``stats_file`` pointer of the table's latest snapshot.
+
+        Prefers the leaf payload (already in Redis); falls back to reading the
+        snapshot JSON from storage. ``None`` when the table has no stats artifact
+        yet (never written, or written before stats existed).
+        """
+        catalog = RedisCatalog()
+        leaf = catalog.get_leaf(self.organization, super_name, simple_name)
+        if not isinstance(leaf, dict):
+            return None
+        payload = leaf.get("payload")
+        if isinstance(payload, dict) and payload.get("stats_file"):
+            return payload["stats_file"]
+        path = leaf.get("path")
+        if not path:
+            return None
+        from supertable.super_table import SuperTable
+        snapshot = SuperTable(
+            super_name, self.organization, create_if_missing=False,
+        ).read_simple_table_snapshot(path)
+        return snapshot.get("stats_file") if isinstance(snapshot, dict) else None
+
+    def _execute_show_stats(
+        self, command, role_name: str,
+    ) -> Tuple[pd.DataFrame, Status, Optional[str]]:
+        """Return the raw contents of a table's latest statistics parquet.
+
+        Reads-never-create and table-level RBAC are enforced (the same gates a
+        SELECT hits); the statistics rows/columns themselves are returned
+        unfiltered. When the table exists but has no stats artifact yet, an empty
+        frame with the stats schema columns is returned (success, not error).
+        """
+        from supertable.data_classes import TableDefinition
+        from supertable.processing import load_stats, STATS_SCHEMA
+
+        super_name = command.super_name
+        simple_name = command.simple_name
+        td = TableDefinition(
+            super_name=super_name,
+            simple_name=simple_name,
+            alias=simple_name,
+            columns=[],
+        )
+
+        # Reads never create catalog entries.
+        try:
+            self._assert_targets_exist([td])
+        except (SuperTableNotFoundError, TableNotFoundError) as e:
+            logger.warning(self._lp(f"[show-stats] target missing: {e}"))
+            return pd.DataFrame(), Status.ERROR, str(e)
+
+        # Table-level RBAC: raises PermissionError if the role cannot read the
+        # table at all. columns=[] means "all columns", which skips column-level
+        # denial — we don't filter the stats output, only gate table access.
+        restrict_read_access(
+            super_name=super_name,
+            organization=self.organization,
+            role_name=role_name,
+            tables=[td],
+            physical_tables=[td],
+        )
+
+        try:
+            stats_file = self._resolve_latest_stats_file(super_name, simple_name)
+            stats_df = load_stats(stats_file, allow_cache=True) if stats_file else None
+        except Exception as e:
+            logger.error(self._lp(f"[show-stats] failed to load stats: {e}"))
+            return pd.DataFrame(), Status.ERROR, str(e)
+
+        if stats_df is None:
+            return pd.DataFrame(columns=list(STATS_SCHEMA.keys())), Status.OK, None
+        return stats_df.to_pandas(), Status.OK, None
+
     def execute(
         self,
         role_name: str,
@@ -120,10 +197,27 @@ class DataReader:
         self.timer = Timer()
         self.plan_stats = PlanStats()
 
-        # Build parser with the correct dialect for the chosen engine.
+        # Classify into an allowed read-path command. Ordinary SELECTs fall
+        # through unchanged; EXPLAIN/SHOW STATS are the two diagnostic
+        # extensions. A recognised-but-malformed command (e.g. SHOW STATS with
+        # no table) returns a clean error rather than raising.
+        try:
+            command = classify_query(self.query, self.super_name)
+        except ValueError as e:
+            logger.warning(self._lp(f"rejected query: {e}"))
+            return pd.DataFrame(), Status.ERROR, str(e)
+
+        # SHOW STATS short-circuits the engine entirely — it returns the raw
+        # statistics artifact, no reflection/estimation/execution.
+        if command.kind is CommandKind.SHOW_STATS:
+            return self._execute_show_stats(command, role_name)
+
+        # Build parser with the correct dialect for the chosen engine. For
+        # EXPLAIN, parse only the inner SELECT so estimation/RBAC/reflection
+        # behave exactly as for the equivalent plain SELECT.
         parser = SQLParser(
             super_name=self.super_name,
-            query=self.query,
+            query=command.sql,
             dialect=engine.dialect,
         )
         tables = parser.get_table_tuples()
@@ -180,11 +274,21 @@ class DataReader:
             self._log_ctx = f"[qid={self.query_plan_manager.query_id} qh={self.query_plan_manager.query_hash}] "
             self.query_plan_manager.original_table = ", ".join(t.simple_name for t in physical_tables) if physical_tables else ""
 
+            # Derive per-table WHERE constraints so the estimator can prune
+            # files via the stats artifact.  Never let this break a read.
+            try:
+                predicate_constraints = parser.get_predicate_constraints()
+            except Exception as pc_err:
+                logger.debug(self._lp(f"[prune] predicate extraction failed: {pc_err}"))
+                predicate_constraints = {}
+
             # 1) ESTIMATE — use physical_tables so CTE aliases are excluded
             estimator = DataEstimator(
                 organization=self.organization,
                 storage=self.storage,
-                tables=physical_tables
+                tables=physical_tables,
+                predicate_constraints=predicate_constraints,
+                plan_stats=self.plan_stats,
             )
             reflection = estimator.estimate()
 
@@ -194,26 +298,13 @@ class DataReader:
             # executors create filtered views for restricted roles.
             reflection.rbac_views = rbac_views
 
-            # --- Dedup-on-read & tombstones: look up table configs and snapshot metadata ---
+            # --- Tombstone (deletion-vector): look up snapshot metadata ------
             try:
                 catalog = RedisCatalog()
                 for td in tables:
-                    tbl_cfg = catalog.get_table_config(
-                        self.organization, td.super_name, td.simple_name,
-                    )
-                    if tbl_cfg and tbl_cfg.get("dedup_on_read"):
-                        pk = tbl_cfg.get("primary_keys", [])
-                        if pk:
-                            # visible_columns: what the user actually asked for.
-                            # Empty list (SELECT *) means "all user columns" — the
-                            # executor will expose everything except __rn__.
-                            reflection.dedup_views[td.alias] = DedupViewDef(
-                                primary_keys=pk,
-                                order_column="__timestamp__",
-                                visible_columns=list(td.columns or []),
-                            )
-
-                    # Tombstone filtering: read from snapshot payload in Redis leaf
+                    # Tombstone filtering: read the deletion-vector pointer from
+                    # the snapshot payload in the Redis leaf.  When present, the
+                    # executor anti-joins the data on __rowid__ against it.
                     payload = None
                     try:
                         leaf = catalog.get_leaf(
@@ -221,15 +312,11 @@ class DataReader:
                         )
                         payload = (leaf or {}).get("payload") if isinstance(leaf, dict) else None
                         if isinstance(payload, dict):
-                            tomb_block = payload.get("tombstones")
-                            if isinstance(tomb_block, dict):
-                                tomb_keys = tomb_block.get("deleted_keys") or []
-                                tomb_pk = tomb_block.get("primary_keys") or []
-                                if tomb_keys and tomb_pk:
-                                    reflection.tombstone_views[td.alias] = TombstoneDef(
-                                        primary_keys=tomb_pk,
-                                        deleted_keys=tomb_keys,
-                                    )
+                            tomb_path = payload.get("tombstone")
+                            if tomb_path:
+                                reflection.tombstone_views[td.alias] = TombstoneDef(
+                                    tombstone_path=tomb_path,
+                                )
                     except Exception as te:
                         logger.debug(self._lp(f"[tombstone] leaf lookup failed for {td.alias}: {te}"))
 
@@ -262,15 +349,23 @@ class DataReader:
                 message = "No parquet files found"
                 return pd.DataFrame(), status, message
 
-            # 2) EXECUTE
+            # 2) EXECUTE.  EXPLAIN is pinned to DuckDB-lite so the plan is
+            # produced cheaply and uniformly (no Pro materialisation / Spark
+            # round trip) and prefixed onto the final rewritten query.
+            exec_engine = engine
+            if command.explain:
+                from supertable.engine.engine_enum import Engine as _EngineEnum
+                exec_engine = _EngineEnum.DUCKDB_LITE
             result_df, engine_used = executor.execute(
-                engine=engine,
+                engine=exec_engine,
                 reflection=reflection,
                 parser=parser,
                 query_manager=self.query_plan_manager,
                 timer=self.timer,
                 plan_stats=self.plan_stats,
                 log_prefix=self._lp(""),
+                explain=command.explain,
+                explain_options=command.explain_options,
             )
             status = Status.OK
         except Exception as e:
@@ -337,8 +432,14 @@ def query_sql(
     the caller can correlate its own audit log to this read record.
     """
     # Safety guard: ensure a LIMIT is present so unbounded queries don't
-    # overwhelm the MCP response payload.
-    sql = _ensure_sql_limit(sql, default_limit=limit)
+    # overwhelm the MCP response payload. Only plain SELECTs take an appended
+    # LIMIT — EXPLAIN output is tiny and SHOW STATS does not accept a LIMIT.
+    try:
+        is_select = classify_query(sql, super_name).kind is CommandKind.SELECT
+    except ValueError:
+        is_select = True
+    if is_select:
+        sql = _ensure_sql_limit(sql, default_limit=limit)
 
     reader = DataReader(
         organization=organization, super_name=super_name, query=sql, source=source,