supertable 2.1.1__tar.gz → 2.2.1__tar.gz

{supertable-2.1.1/supertable.egg-info → supertable-2.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supertable
-Version: 2.1.1
+Version: 2.2.1
 Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
 Author: Levente Kupas
 Author-email: Levente Kupas <lkupas@kladnasoft.com>

{supertable-2.1.1 → supertable-2.2.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "supertable"
-version = "2.1.1"
+version = "2.2.1"
 description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
 readme = "README.md"
 requires-python = ">=3.10"

{supertable-2.1.1 → supertable-2.2.1}/setup.py

@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
 
 setup(
     name="supertable",
-    version="2.1.1",
+    version="2.2.1",
     description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
     long_description=long_description,
     long_description_content_type="text/markdown",

{supertable-2.1.1 → supertable-2.2.1}/supertable/__init__.py

@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
 project documentation for the full API surface.
 """
 
-__version__ = "2.1.1"
+__version__ = "2.2.1"
 
 # Re-export the core public surface so users can do ``from supertable import …``
 # instead of remembering submodule paths.
@@ -39,6 +39,11 @@ from supertable.super_pipe import SuperPipe
 from supertable.redis_catalog import RedisCatalog
 from supertable.rbac.role_manager import RoleManager
 from supertable.rbac.user_manager import UserManager
+from supertable.errors import (
+    SupertableLookupError,
+    SuperTableNotFoundError,
+    TableNotFoundError,
+)
 
 __all__ = [
     "__version__",
@@ -55,4 +60,7 @@ __all__ = [
     "RedisCatalog",
     "RoleManager",
     "UserManager",
+    "SupertableLookupError",
+    "SuperTableNotFoundError",
+    "TableNotFoundError",
 ]

{supertable-2.1.1 → supertable-2.2.1}/supertable/config/settings.py

@@ -282,6 +282,21 @@ class Settings:
     SUPERTABLE_SHARE_PRESIGN_TTL: int = 14400                # SUPERTABLE_SHARE_PRESIGN_TTL (seconds, default 4h)
     SUPERTABLE_SHARE_REFRESH_BUFFER: int = 600               # SUPERTABLE_SHARE_REFRESH_BUFFER (seconds, refresh 10min before expiry)
 
+    # ── Storage GC (deferred deletion of sunset parquets + old snapshot JSONs) ──
+    # Sunset parquet files and old snapshot JSONs are never deleted by the
+    # writer directly. When either flag is enabled, the writer XADDs paths
+    # to a per-table Redis STREAM after a successful leaf-CAS. A separate
+    # cleaner daemon (``python -m supertable.gc.cleaner``) drains entries
+    # older than ``SUPERTABLE_GC_DELAY_SEC`` and calls ``storage.delete()``.
+    # The delay window avoids the race where an in-flight reader's
+    # ``parquet_scan([...])`` resolves a file path right before the writer
+    # deletes it. Defaults below preserve today's behaviour (off, unlimited).
+    SUPERTABLE_SNAPSHOT_RETENTION: int = 0                   # SUPERTABLE_SNAPSHOT_RETENTION (0 = keep all snapshot JSONs)
+    SUPERTABLE_SUNSET_GC_ENABLED: bool = False               # SUPERTABLE_SUNSET_GC_ENABLED (delete sunset parquets via queue)
+    SUPERTABLE_GC_DELAY_SEC: int = 1800                      # SUPERTABLE_GC_DELAY_SEC (min age before cleaner deletes)
+    SUPERTABLE_GC_SLEEP_SEC: int = 60                        # SUPERTABLE_GC_SLEEP_SEC (cleaner loop sleep)
+    SUPERTABLE_GC_BATCH_SIZE: int = 500                      # SUPERTABLE_GC_BATCH_SIZE (max entries processed per table per tick)
+
 
     # ── Convenience properties ───────────────────────────────────────
 
@@ -528,6 +543,13 @@ def _build_settings() -> Settings:
         # Data Sharing
         SUPERTABLE_SHARE_PRESIGN_TTL=_env_int("SUPERTABLE_SHARE_PRESIGN_TTL", 14400),
         SUPERTABLE_SHARE_REFRESH_BUFFER=_env_int("SUPERTABLE_SHARE_REFRESH_BUFFER", 600),
+
+        # ── Storage GC ───────────────────────────────────────────────
+        SUPERTABLE_SNAPSHOT_RETENTION=_env_int("SUPERTABLE_SNAPSHOT_RETENTION", 0),
+        SUPERTABLE_SUNSET_GC_ENABLED=_env_bool("SUPERTABLE_SUNSET_GC_ENABLED", False),
+        SUPERTABLE_GC_DELAY_SEC=_env_int("SUPERTABLE_GC_DELAY_SEC", 1800),
+        SUPERTABLE_GC_SLEEP_SEC=_env_int("SUPERTABLE_GC_SLEEP_SEC", 60),
+        SUPERTABLE_GC_BATCH_SIZE=_env_int("SUPERTABLE_GC_BATCH_SIZE", 500),
     )
 
 

{supertable-2.1.1 → supertable-2.2.1}/supertable/data_reader.py

@@ -10,6 +10,7 @@ from typing import Optional, Tuple, Any, List, Dict
 import pandas as pd
 
 from supertable.config.defaults import logger
+from supertable.errors import SuperTableNotFoundError, TableNotFoundError
 from supertable.storage.storage_factory import get_storage
 from supertable.storage.storage_interface import StorageInterface
 from supertable.utils.timer import Timer
@@ -59,6 +60,44 @@ class DataReader:
     def _lp(self, msg: str) -> str:
         return f"{self._log_ctx}{msg}"
 
+    def _assert_targets_exist(self, physical_tables) -> None:
+        """Fail fast if any referenced (super, simple) is missing in Redis.
+
+        The read path must never create catalog entries as a side effect
+        of resolving a query. ``SuperTable`` / ``SimpleTable``
+        constructors used to do exactly that for callers that didn't pass
+        ``create_if_missing=False`` — this guard is the SDK-level
+        invariant that says "reads cannot mint tables".
+
+        Raises:
+            SuperTableNotFoundError: when the supertable's
+                ``meta:root`` pointer is missing.
+            TableNotFoundError: when the simple table's
+                ``meta:leaf:doc:{simple}`` pointer is missing.
+        """
+        if not physical_tables:
+            return
+        # One catalog handle for the whole loop — cheaper than letting
+        # each .exists() call open a fresh connection.
+        catalog = RedisCatalog()
+        # Dedup by (super, simple) — SQL may mention the same table
+        # multiple times via different aliases.
+        seen = set()
+        for td in physical_tables:
+            super_name = td.super_name
+            simple_name = td.simple_name
+            if not super_name or not simple_name:
+                continue
+            key = (super_name, simple_name)
+            if key in seen:
+                continue
+            seen.add(key)
+            if not catalog.root_exists(self.organization, super_name):
+                raise SuperTableNotFoundError(self.organization, super_name)
+            if not catalog.leaf_exists(self.organization, super_name, simple_name):
+                raise TableNotFoundError(
+                    self.organization, super_name, simple_name
+                )
 
     def execute(
         self,
@@ -80,7 +119,32 @@ class DataReader:
         tables = parser.get_table_tuples()
         physical_tables = parser.get_physical_tables()
 
-        # RBAC check — also returns per-alias column/row filter definitions
+        # Read-path policy: reads never create. Verify every referenced
+        # (super, simple) exists in the Redis catalog **before** anything
+        # downstream — RBAC, estimator, or the executor — gets a chance
+        # to side-effect-bootstrap them.
+        #
+        # ORDERING MATTERS: ``restrict_read_access`` (called next) builds
+        # ``RoleManager(super_name=..., organization=...)`` which boots
+        # RBAC role storage in Redis for the supertable if it doesn't
+        # exist. Running the RBAC check first against a missing
+        # supertable would silently mint the RBAC scaffold before our
+        # existence check fires. Pre-flight FIRST.
+        #
+        # The check runs in its own try block so SuperTable/TableNotFound
+        # convert to the standard (empty_df, Status.ERROR, message)
+        # return — we don't want to raise these into the caller, but we
+        # DO want to keep ``restrict_read_access``'s PermissionError
+        # raising naturally (legacy behaviour API layers depend on for
+        # 403 translation).
+        try:
+            self._assert_targets_exist(physical_tables)
+        except (SuperTableNotFoundError, TableNotFoundError) as e:
+            logger.warning(self._lp(f"target missing: {e}"))
+            return pd.DataFrame(), Status.ERROR, str(e)
+
+        # RBAC check — also returns per-alias column/row filter definitions.
+        # PermissionError propagates to the caller (legacy behaviour).
         rbac_views = restrict_read_access(
             super_name=self.super_name,
             organization=self.organization,