supertable 2.0.5__tar.gz → 2.0.7__tar.gz

{supertable-2.0.5/supertable.egg-info → supertable-2.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supertable
-Version: 2.0.5
+Version: 2.0.7
 Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
 Author: Levente Kupas
 Author-email: Levente Kupas <lkupas@kladnasoft.com>

{supertable-2.0.5 → supertable-2.0.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "supertable"
-version = "2.0.5"
+version = "2.0.7"
 description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
 readme = "README.md"
 requires-python = ">=3.10"

{supertable-2.0.5 → supertable-2.0.7}/setup.py

@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
 
 setup(
     name="supertable",
-    version="2.0.5",
+    version="2.0.7",
     description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
     long_description=long_description,
     long_description_content_type="text/markdown",

{supertable-2.0.5 → supertable-2.0.7}/supertable/__init__.py

@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
 project documentation for the full API surface.
 """
 
-__version__ = "2.0.5"
+__version__ = "2.0.7"
 
 # Re-export the core public surface so users can do ``from supertable import …``
 # instead of remembering submodule paths.

{supertable-2.0.5 → supertable-2.0.7}/supertable/audit/admin.py

@@ -9,7 +9,7 @@ Persistence
 -----------
 Stored in Redis as a HASH at:
 
-    supertable:{org}:audit:config
+    supertable:{org}:system:audit:config
 
 Fields
 ------

{supertable-2.0.5 → supertable-2.0.7}/supertable/audit/logger.py

@@ -408,7 +408,7 @@ _LOGGERS: Dict[str, "AuditLogger | NullAuditLogger"] = {}
 _LOGGERS_LOCK = threading.Lock()
 
 # Per-org config cache: org → (config, expires_at_seconds).
-# Resolved against env defaults + Redis override (supertable:{org}:audit:config).
+# Resolved against env defaults + Redis override (supertable:{org}:system:audit:config).
 _ORG_CFG_CACHE: Dict[str, "tuple[AuditConfig, float]"] = {}
 _ORG_CFG_TTL_S: float = 30.0  # toggle takes effect within this many seconds
 
@@ -416,7 +416,7 @@ _ORG_CFG_TTL_S: float = 30.0  # toggle takes effect within this many seconds
 def _resolve_config_for(organization: str) -> AuditConfig:
     """Resolve the effective AuditConfig for *organization*.
 
-    Merges the Redis override at ``supertable:{org}:audit:config`` over the
+    Merges the Redis override at ``supertable:{org}:system:audit:config`` over the
     env-var defaults.  Cached for _ORG_CFG_TTL_S seconds.
     """
     now = time.time()

{supertable-2.0.5 → supertable-2.0.7}/supertable/audit/retention.py

@@ -25,17 +25,13 @@ import re
 from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, List, Optional
 
-logger = logging.getLogger(__name__)
+from supertable import redis_keys as RK
 
-# ---------------------------------------------------------------------------
-# Redis key for legal hold runtime override
-# ---------------------------------------------------------------------------
-
-_LEGAL_HOLD_KEY_TEMPLATE = "supertable:{org}:audit:legal_hold"
+logger = logging.getLogger(__name__)
 
 
 def _legal_hold_key(org: str) -> str:
-    return _LEGAL_HOLD_KEY_TEMPLATE.replace("{org}", org)
+    return RK.audit_legal_hold(org)
 
 
 # ---------------------------------------------------------------------------

{supertable-2.0.5 → supertable-2.0.7}/supertable/audit/tests/test_retention.py

@@ -28,10 +28,13 @@ from supertable.audit import retention
 
 class TestLegalHoldKey:
     def test_template_substitution(self) -> None:
-        assert retention._legal_hold_key("acme") == "supertable:acme:audit:legal_hold"
+        assert retention._legal_hold_key("acme") == "supertable:acme:system:audit:legal_hold"
 
-    def test_handles_special_characters(self) -> None:
-        assert retention._legal_hold_key("acme-EU") == "supertable:acme-EU:audit:legal_hold"
+    def test_handles_hyphens(self) -> None:
+        # v2 _safe() requires the org-segment regex
+        # ^[a-z0-9][a-z0-9_-]{0,63}$ — uppercase letters are rejected,
+        # hyphens + lowercase alphanumerics are allowed.
+        assert retention._legal_hold_key("acme-eu") == "supertable:acme-eu:system:audit:legal_hold"
 
 
 class TestParsePartitionDate:
@@ -171,7 +174,7 @@ class TestSetLegalHold:
         assert result == {"ok": True, "legal_hold": True, "organization": "acme"}
 
         fake_client.set.assert_called_once_with(
-            "supertable:acme:audit:legal_hold", "1"
+            "supertable:acme:system:audit:legal_hold", "1"
         )
         assert audit_calls and audit_calls[0]["organization"] == "acme"
 

{supertable-2.0.5 → supertable-2.0.7}/supertable/audit/writer_redis.py

@@ -6,7 +6,7 @@ Provides real-time queryability (XRANGE), consumer groups for external
 SIEM tools (Splunk, Sentinel, ELK), and TTL-based eviction.
 
 Each organization gets its own stream:
-    supertable:{org}:audit:stream
+    supertable:{org}:system:audit:stream
 
 The internal archival consumer group ("__archival__") is created
 automatically. External SIEM consumer groups are managed via the

{supertable-2.0.5 → supertable-2.0.7}/supertable/config/settings.py

@@ -261,7 +261,7 @@ class Settings:
 
     # ── Audit ────────────────────────────────────────────────────────
     # Audit is OFF by default.  Enable per-organization in the WebUI
-    # /ui/audit → Compliance tab (persisted at supertable:{org}:audit:config),
+    # /ui/audit → Compliance tab (persisted at supertable:{org}:system:audit:config),
     # or globally via the SUPERTABLE_AUDIT_ENABLED env var.
     SUPERTABLE_AUDIT_ENABLED: bool = False                   # SUPERTABLE_AUDIT_ENABLED
     SUPERTABLE_AUDIT_RETENTION_DAYS: int = 2555              # SUPERTABLE_AUDIT_RETENTION_DAYS (~7 years)

{supertable-2.0.5 → supertable-2.0.7}/supertable/data_writer.py

@@ -535,8 +535,12 @@ class DataWriter:
         # guaranteed to reach Redis before this scope closes.
         try:
             if stats_payload is not None:
+                # Monitoring is org-wide as of SDK 2.2.0 — record the
+                # touched supertable in the payload's ``supertables``
+                # field. A DataWriter only touches one supertable, but
+                # the list shape is uniform with cross-sup events.
+                stats_payload["supertables"] = [self.super_table.super_name]
                 with MonitoringWriter(
-                    super_name=self.super_table.super_name,
                     organization=self.super_table.organization,
                     monitor_type="writes",
                 ) as monitor:

{supertable-2.0.5 → supertable-2.0.7}/supertable/meta_reader.py

@@ -154,7 +154,7 @@ class MetaReader:
                 count=1000,
             ):
                 key_str = key if isinstance(key, str) else key.decode('utf-8')
-                table_name = key_str.rsplit("meta:leaf:", 1)[-1]
+                table_name = key_str.rsplit("meta:leaf:doc:", 1)[-1]
                 if table_name and table_name not in seen:
                     seen.add(table_name)
                     tables.append(table_name)
@@ -507,7 +507,10 @@ def list_supers(organization: str, role_name: str) -> List[str]:
 
     items = _get_redis_items(pattern)
     for item in items:
-        super_name = item.split(':')[2]
+        parsed = RK.parse_lake_key(item)
+        if parsed is None:
+            continue
+        _, super_name = parsed
         try:
             check_meta_access(
                 super_name=super_name,

{supertable-2.0.5 → supertable-2.0.7}/supertable/mirroring/mirror_formats.py

@@ -40,7 +40,7 @@ class MirrorFormats:
     Redis-backed format mirror configuration and dispatch.
 
     Storage:
-      - Redis key: supertable:{org}:{super}:meta:mirrors
+      - Redis key: supertable:{org}:lakes:{super}:meta:mirrors
         value: {"formats": ["DELTA", ...], "ts": <epoch_ms>}
     """
 

{supertable-2.0.5 → supertable-2.0.7}/supertable/monitoring_writer.py

@@ -109,19 +109,21 @@ class NullMonitoringLogger:
 @dataclass(frozen=True)
 class _MonitorKey:
     organization: str
-    super_name: str
     monitor_type: str
 
     @property
     def path_key(self) -> str:
-        # Matches your log style: "org/super/monitor_type"
-        return f"{self.organization}/{self.super_name}/{self.monitor_type}"
+        # Matches the log style: "org/monitor_type"
+        return f"{self.organization}/{self.monitor_type}"
 
     @property
     def redis_list_key(self) -> str:
-        # Namespaced under the supertable scope:
-        #   supertable:{org}:{sup}:monitor:{monitor_type}
-        return RK.monitor(self.organization, self.super_name, self.monitor_type)
+        # v2.2: org-wide monitoring lives at position 2.
+        #   supertable:{org}:monitor:{monitor_type}
+        # Cross-supertable queries record exactly one canonical entry
+        # here; attribution is preserved in the payload's
+        # ``supertables: [str]`` field.
+        return RK.monitor(self.organization, self.monitor_type)
 
 
 class _AsyncMonitoringLogger:
@@ -455,7 +457,6 @@ def _monitoring_enabled() -> bool:
 
 def get_monitoring_logger(
     *,
-    super_name: str,
     organization: str,
     monitor_type: str = "plans",
     redis_connector: Optional["RedisConnector"] = None,
@@ -464,11 +465,16 @@ def get_monitoring_logger(
     Return a monitoring logger (context manager + log_metric + request_flush + debug fields).
 
     This function never raises; on any failure it returns a NullMonitoringLogger.
+
+    Per-supertable attribution is encoded **in the payload**, not in
+    the Redis key — callers include a ``supertables: [str]`` field in
+    each payload passed to ``log_metric``. The org-level shape lets
+    cross-supertable queries record a single canonical entry.
     """
     if not _monitoring_enabled():
         return NullMonitoringLogger()
 
-    key = _MonitorKey(organization=organization, super_name=super_name, monitor_type=monitor_type)
+    key = _MonitorKey(organization=organization, monitor_type=monitor_type)
     cache_key = key.path_key
 
     try:
@@ -490,33 +496,38 @@ def get_monitoring_logger(
 
 class MonitoringWriter:
     """
-    Backwards-compatible facade.
+    Monitoring façade — org-level writer (SDK 2.2.0+).
 
-    Supported usage patterns:
-        mon = MonitoringWriter(...)
-        mon.log_metric({...})
+    Usage:
+        mon = MonitoringWriter(organization=org, monitor_type="plans")
+        mon.log_metric({"supertables": ["sales", "customers"], ...})
         mon.request_flush()
 
-    And context-manager style:
-        with MonitoringWriter(...) as mon:
-            mon.log_metric({...})
+    Context-manager style:
+        with MonitoringWriter(organization=org, monitor_type="writes") as mon:
+            mon.log_metric({"supertables": ["sales"], ...})
+
+    Per-supertable attribution: the Redis key is org-level
+    (``supertable:{org}:monitor:{monitor_type}``). Callers include a
+    ``supertables: [str]`` field in each payload — the full list of
+    supertables the event touches. Cross-supertable queries (JOINs,
+    multi-target writes) pass the full list. Events not tied to any
+    supertable pass ``supertables: []`` (or omit it — the writer
+    will default to an empty list).
 
-    It also forwards debug attributes used in examples (queue_stats, etc.).
+    Also forwards debug attributes used in examples (queue_stats, etc.).
     """
 
     def __init__(
         self,
         *,
-        super_name: str,
         organization: str,
         monitor_type: str = "plans",
         redis_connector: Optional["RedisConnector"] = None,
     ):
-        self.super_name = super_name
         self.organization = organization
         self.monitor_type = monitor_type
         self._logger: MonitoringLogger = get_monitoring_logger(
-            super_name=super_name,
             organization=organization,
             monitor_type=monitor_type,
             redis_connector=redis_connector,

{supertable-2.0.5 → supertable-2.0.7}/supertable/plan_extender.py

@@ -123,10 +123,12 @@ def extend_execution_plan(
         logger.error("Failed to build monitoring stats payload: %s", e)
         return  # nothing else to do safely
 
-    # Log the metric (buffered; background writer flushes)
+    # Log the metric (buffered; background writer flushes).
+    # Monitoring is org-wide as of SDK 2.2.0 — the touched supertable
+    # is recorded in the payload's ``supertables: [str]`` field.
     try:
+        stats["supertables"] = [query_plan_manager.super_name]
         with MonitoringWriter(
-            super_name=query_plan_manager.super_name,
             organization=query_plan_manager.organization,
             monitor_type="plans",
         ) as monitor:

{supertable-2.0.5 → supertable-2.0.7}/supertable/rbac/tests/test_rbac.py

@@ -1659,7 +1659,7 @@ class TestKeyNamespace(unittest.TestCase):
             RK.rbac_role_type_index("o", "s", "admin"),
         ]
         for k in keys:
-            self.assertTrue(k.startswith("supertable:o:s:rbac:"), f"Bad key: {k}")
+            self.assertTrue(k.startswith("supertable:o:lakes:s:rbac:"), f"Bad key: {k}")
 
     def test_rbac_keys_do_not_collide(self):
         """All 8 key patterns for same org/sup produce distinct keys."""

{supertable-2.0.5 → supertable-2.0.7}/supertable/redis_catalog.py

@@ -117,7 +117,7 @@ return v
     #   ARGV[1] role_id
     #   ARGV[2] now_ms (string)
     #   ARGV[3] role_name_lower (or "")
-    #   ARGV[4] user_doc_key_prefix  (e.g. "supertable:{org}:{sup}:rbac:users:doc:")
+    #   ARGV[4] user_doc_key_prefix  (e.g. "supertable:{org}:lakes:{sup}:rbac:users:doc:")
     # KEYS layout:
     #   KEYS[1] role_doc_key
     #   KEYS[2] role_index_key
@@ -264,7 +264,7 @@ return 1
             timeout_s: int = 30,
     ) -> Optional[str]:
         """Acquire lock for staging/pipe operations:
-            supertable:{org}:{sup}:lock:stage:{stage_name}
+            supertable:{org}:lakes:{sup}:lock:stage:doc:{stage_name}
         """
         return self._locker.acquire(RK.lock_stage(org, sup, stage_name), ttl_s=ttl_s, timeout_s=timeout_s)
 
@@ -343,10 +343,10 @@ return 1
                 cursor, keys = self.r.scan(cursor=cursor, match=pattern, count=500)
                 for k in keys:
                     ks = k if isinstance(k, str) else k.decode("utf-8")
-                    parts = ks.split(":")
-                    if len(parts) < 4:
+                    parsed = RK.parse_lake_key(ks)
+                    if parsed is None:
                         continue
-                    sup_name = parts[2]
+                    _, sup_name = parsed
                     if sup_name == source_sup:
                         continue
                     try:
@@ -685,8 +685,8 @@ return 1
         role_name = self.r.hget(key, "role_name") or ""
         if isinstance(role_name, bytes):
             role_name = role_name.decode("utf-8")
-        # rbac_user_doc(..., "") yields the canonical "...:rbac:users:doc:" prefix.
-        user_doc_key_prefix = RK.rbac_user_doc(org, sup, "")
+        # The Lua script appends each user_id to this prefix.
+        user_doc_key_prefix = RK.rbac_user_doc_prefix(org, sup)
         result = self._rbac_delete_role(
             keys=[
                 key,
@@ -954,7 +954,7 @@ return 1
     # ------------- Listings via SCAN -------------
 
     def scan_leaf_keys(self, org: str, sup: str, count: int = 1000) -> Iterator[str]:
-        """Yields full Redis keys: supertable:{org}:{sup}:meta:leaf:* (replica-aware)."""
+        """Yields full Redis keys: supertable:{org}:lakes:{sup}:meta:leaf:doc:* (replica-aware)."""
         info = self._resolve_replica_info(org, sup)
         effective_sup = info[0] if info else sup
         allowed = info[1] if info else None
@@ -967,7 +967,7 @@ return 1
                 for k in keys:
                     ks = k if isinstance(k, str) else k.decode('utf-8')
                     if allowed:
-                        simple = ks.rsplit("meta:leaf:", 1)[-1]
+                        simple = ks.rsplit("meta:leaf:doc:", 1)[-1]
                         if simple not in allowed:
                             continue
                     yield ks
@@ -1003,7 +1003,7 @@ return 1
                 continue
             try:
                 obj = json.loads(raw)
-                simple = k.rsplit("meta:leaf:", 1)[-1]
+                simple = k.rsplit("meta:leaf:doc:", 1)[-1]
                 yield {
                     "simple": simple,
                     "version": int(obj.get("version", -1)),
@@ -1216,44 +1216,22 @@ return 1
             return None
 
     def list_stagings(self, org: str, sup: str, *, count: int = 1000) -> List[str]:
-        """List staging names. Prefers the staging index set; falls back to SCAN."""
+        """List staging names from the staging index set."""
         if not (org and sup):
             return []
         try:
-            names = list(self.r.smembers(RK.staging_index(org, sup)) or [])
-            if names:
-                return sorted({(n if isinstance(n, str) else n.decode('utf-8')) for n in names if n})
+            names = self.r.smembers(RK.staging_index(org, sup)) or set()
         except redis.RedisError as e:
             logger.error(f"[redis-catalog] list_stagings smembers error: {e}")
-
-        # Fallback (migration/back-compat): scan exact staging meta keys.
-        pattern = RK.staging_pattern(org, sup)
-        seen = set()
-        cursor = 0
-        try:
-            while True:
-                cursor, keys = self.r.scan(cursor=cursor, match=pattern, count=max(1, int(count)))
-                for k in keys or []:
-                    kk = k if isinstance(k, str) else k.decode("utf-8")
-                    if ":pipe:" in kk:
-                        continue
-                    if kk.endswith(":meta"):
-                        continue
-                    name = kk.rsplit("meta:staging:", 1)[-1]
-                    if name:
-                        seen.add(name)
-                if cursor == 0:
-                    break
-        except redis.RedisError as e:
-            logger.error(f"[redis-catalog] list_stagings scan error: {e}")
-        return sorted(seen)
+            return []
+        return sorted({(n if isinstance(n, str) else n.decode('utf-8')) for n in names if n})
 
     def delete_staging_meta(self, org: str, sup: str, staging_name: str, *, count: int = 1000) -> int:
         """Delete staging meta and *all* related keys under the staging prefix.
 
         This removes the staging from the staging index set, deletes the staging meta key,
         and deletes any keys matching:
-            supertable:{org}:{sup}:meta:staging:{staging_name}:*
+            supertable:{org}:lakes:{sup}:meta:staging:doc:{staging_name}:*
         Returns number of keys deleted (best-effort; does not include SREM).
         """
         if not (org and sup and staging_name):
@@ -1393,7 +1371,7 @@ return 1
             return 0
 
     # ========================================================================= #
-    # Spark Thrift cluster management (org-scoped: supertable:{org}:spark:thrifts)
+    # Spark Thrift cluster management (org-scoped: supertable:{org}:system:spark:thrifts)
     # ========================================================================= #
 
     def register_spark_cluster(self, org: str, cluster_id: str, config: Dict[str, Any]) -> None:
@@ -1489,7 +1467,7 @@ return 1
         return candidates[0]
 
     # ========================================================================= #
-    # Spark Plug management (org-scoped: supertable:{org}:spark:plugs)
+    # Spark Plug management (org-scoped: supertable:{org}:system:spark:plugs)
     # ========================================================================= #
 
     def register_spark_plug(self, org: str, plug_id: str, config: Dict[str, Any]) -> None:

{supertable-2.0.5 → supertable-2.0.7}/supertable/redis_infra.py

@@ -177,7 +177,7 @@ class _FallbackCatalog:
                 continue
             try:
                 obj = json.loads(raw if isinstance(raw, str) else raw.decode("utf-8"))
-                simple = k.rsplit("meta:leaf:", 1)[-1]
+                simple = k.rsplit("meta:leaf:doc:", 1)[-1]
                 yield {
                     "simple": simple,
                     "version": int(obj.get("version", -1)),