supertable 2.0.0__tar.gz → 2.0.2__tar.gz

{supertable-2.0.0 → supertable-2.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: supertable
-Version: 2.0.0
+Version: 2.0.2
 Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
 Author: Levente Kupas
 Author-email: Levente Kupas <lkupas@kladnasoft.com>

{supertable-2.0.0 → supertable-2.0.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "supertable"
-version = "2.0.0"
+version = "2.0.2"
 description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
 readme = "README.md"
 requires-python = ">=3.10"

{supertable-2.0.0 → supertable-2.0.2}/setup.py

@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
 
 setup(
     name="supertable",
-    version="2.0.0",
+    version="2.0.2",
     description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
     long_description=long_description,
     long_description_content_type="text/markdown",

{supertable-2.0.0 → supertable-2.0.2}/supertable/__init__.py

@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
 project documentation for the full API surface.
 """
 
-__version__ = "2.0.0"
+__version__ = "2.0.2"
 
 # Re-export the core public surface so users can do ``from supertable import …``
 # instead of remembering submodule paths.

{supertable-2.0.0 → supertable-2.0.2}/supertable/audit/consumers.py

@@ -21,7 +21,7 @@ logger = logging.getLogger(__name__)
 def create_consumer(organization: str, group_name: str, start_from: str = "$") -> Dict[str, Any]:
     """Create an external SIEM consumer group on the audit stream."""
     try:
-        from supertable.server_common import redis_client
+        from supertable.redis_infra import redis_client
         from supertable.audit.writer_redis import RedisAuditWriter
         writer = RedisAuditWriter(redis_client, organization, "", maxlen=0)
         ok = writer.create_consumer_group(group_name, start_from)
@@ -34,7 +34,7 @@ def create_consumer(organization: str, group_name: str, start_from: str = "$") -
 def delete_consumer(organization: str, group_name: str) -> Dict[str, Any]:
     """Remove an external SIEM consumer group."""
     try:
-        from supertable.server_common import redis_client
+        from supertable.redis_infra import redis_client
         from supertable.audit.writer_redis import RedisAuditWriter
         writer = RedisAuditWriter(redis_client, organization, "", maxlen=0)
         ok = writer.delete_consumer_group(group_name)
@@ -47,7 +47,7 @@ def delete_consumer(organization: str, group_name: str) -> Dict[str, Any]:
 def list_consumers(organization: str) -> List[Dict[str, Any]]:
     """List all consumer groups with lag info."""
     try:
-        from supertable.server_common import redis_client
+        from supertable.redis_infra import redis_client
         from supertable.audit.writer_redis import RedisAuditWriter
         writer = RedisAuditWriter(redis_client, organization, "", maxlen=0)
         return writer.list_consumer_groups()

{supertable-2.0.0 → supertable-2.0.2}/supertable/audit/logger.py

@@ -132,7 +132,7 @@ class AuditLogger:
     def _init_redis(self) -> None:
         """Initialize Redis Stream writer."""
         try:
-            from supertable.server_common import redis_client
+            from supertable.redis_infra import redis_client
             from supertable.audit.writer_redis import RedisAuditWriter
             self._redis_writer = RedisAuditWriter(
                 redis_client=redis_client,

{supertable-2.0.0 → supertable-2.0.2}/supertable/audit/reader.py

@@ -93,7 +93,7 @@ def _query_redis(
 ) -> List[Dict[str, Any]]:
     """Query recent events from the Redis Stream."""
     try:
-        from supertable.server_common import redis_client
+        from supertable.redis_infra import redis_client
         from supertable.audit.writer_redis import RedisAuditWriter
         writer = RedisAuditWriter(redis_client, organization, "", maxlen=0)
         return writer.query(start_ms=start_ms, end_ms=end_ms, count=limit)

{supertable-2.0.0 → supertable-2.0.2}/supertable/audit/retention.py

@@ -86,7 +86,7 @@ def is_legal_hold_active(organization: str) -> bool:
     """
     # Try Redis first (runtime override)
     try:
-        from supertable.server_common import redis_client
+        from supertable.redis_infra import redis_client
         val = redis_client.get(_legal_hold_key(organization))
         if val is not None:
             decoded = val if isinstance(val, str) else val.decode("utf-8")
@@ -128,7 +128,7 @@ def set_legal_hold(enabled: bool, organization: str = "") -> Dict[str, Any]:
 
     # Persist to Redis
     try:
-        from supertable.server_common import redis_client
+        from supertable.redis_infra import redis_client
         redis_client.set(_legal_hold_key(organization), "1" if enabled else "0")
     except Exception as e:
         logger.error("[audit-retention] Failed to persist legal hold to Redis: %s", e)

{supertable-2.0.0 → supertable-2.0.2}/supertable/audit/tests/test_retention.py

@@ -83,7 +83,7 @@ class TestIsLegalHoldActive:
     ) -> None:
         fake_client = _make_redis_client(get_value=redis_value)
         fake_module = SimpleNamespace(redis_client=fake_client)
-        monkeypatch.setitem(__import__("sys").modules, "supertable.server_common", fake_module)
+        monkeypatch.setitem(__import__("sys").modules, "supertable.redis_infra", fake_module)
         assert retention.is_legal_hold_active("acme") is True
 
     @pytest.mark.parametrize("redis_value", [b"0", "false", b"no"])
@@ -92,7 +92,7 @@ class TestIsLegalHoldActive:
     ) -> None:
         fake_client = _make_redis_client(get_value=redis_value)
         fake_module = SimpleNamespace(redis_client=fake_client)
-        monkeypatch.setitem(__import__("sys").modules, "supertable.server_common", fake_module)
+        monkeypatch.setitem(__import__("sys").modules, "supertable.redis_infra", fake_module)
         assert retention.is_legal_hold_active("acme") is False
 
     def test_falls_back_to_settings_when_redis_returns_none(
@@ -102,7 +102,7 @@ class TestIsLegalHoldActive:
         fake_server_common = SimpleNamespace(redis_client=fake_client)
         monkeypatch.setitem(
             __import__("sys").modules,
-            "supertable.server_common",
+            "supertable.redis_infra",
             fake_server_common,
         )
 
@@ -125,7 +125,7 @@ class TestIsLegalHoldActive:
                 raise RuntimeError("redis unreachable")
 
         monkeypatch.setitem(
-            __import__("sys").modules, "supertable.server_common", BoomModule()
+            __import__("sys").modules, "supertable.redis_infra", BoomModule()
         )
 
         # Make settings import fail by removing the attribute
@@ -155,7 +155,7 @@ class TestSetLegalHold:
     ) -> None:
         fake_client = _make_redis_client()
         fake_module = SimpleNamespace(redis_client=fake_client)
-        monkeypatch.setitem(__import__("sys").modules, "supertable.server_common", fake_module)
+        monkeypatch.setitem(__import__("sys").modules, "supertable.redis_infra", fake_module)
 
         # Stub out the audit emit chain so the test never touches the real logger.
         import supertable.audit as audit_pkg
@@ -181,7 +181,7 @@ class TestSetLegalHold:
         fake_client = MagicMock()
         fake_client.set.side_effect = RuntimeError("redis down")
         fake_module = SimpleNamespace(redis_client=fake_client)
-        monkeypatch.setitem(__import__("sys").modules, "supertable.server_common", fake_module)
+        monkeypatch.setitem(__import__("sys").modules, "supertable.redis_infra", fake_module)
 
         result = retention.set_legal_hold(False, organization="acme")
         assert result["ok"] is False

{supertable-2.0.0 → supertable-2.0.2}/supertable/monitoring_writer.py

@@ -425,6 +425,14 @@ def _evict_oldest_monitor() -> None:
 
     Strategy: evict the first key (oldest insertion in dict-order, Python 3.7+).
     Signal the evicted logger's worker thread to stop so it doesn't leak.
+
+    Note: we intentionally do NOT close the evicted logger's Redis client.
+    Since supertable.redis_connector.create_redis_client caches one
+    redis.Redis (and one ConnectionPool) per effective RedisOptions, that
+    client is shared with every other monitor / catalog / writer in this
+    process.  Disconnecting it here would tear down their connections too.
+    Call supertable.redis_connector.close_all_redis_clients() at process
+    shutdown if a clean teardown is required.
     """
     if not _MONITORS:
         return

{supertable-2.0.0 → supertable-2.0.2}/supertable/redis_connector.py

@@ -2,9 +2,10 @@
 from __future__ import annotations
 
 import os
+import threading
 import time
 from dataclasses import dataclass, field
-from typing import List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 
 
 import redis
@@ -102,10 +103,47 @@ class RedisOptions:
         object.__setattr__(self, "sentinel_strict", sentinel_strict)
 
 
+# Process-level cache: identical RedisOptions → identical client (and pool).
+# Multiple writer/catalog instances share one ConnectionPool instead of each
+# constructing a new redis.Redis (which would leak its own pool until GC).
+_CLIENT_CACHE: Dict[Tuple[Any, ...], redis.Redis] = {}
+_CLIENT_CACHE_LOCK = threading.Lock()
+
+
+def _options_cache_key(opts: RedisOptions) -> Tuple[Any, ...]:
+    return (
+        opts.host,
+        opts.port,
+        opts.db,
+        opts.password,
+        opts.use_ssl,
+        opts.is_sentinel,
+        tuple(opts.sentinel_hosts),
+        opts.sentinel_master,
+        opts.sentinel_password,
+        opts.sentinel_strict,
+    )
+
+
 def create_redis_client(options: Optional[RedisOptions] = None) -> redis.Redis:
     opts = options or RedisOptions()
     decode_responses = True
 
+    cache_key = _options_cache_key(opts)
+    cached = _CLIENT_CACHE.get(cache_key)
+    if cached is not None:
+        return cached
+
+    with _CLIENT_CACHE_LOCK:
+        cached = _CLIENT_CACHE.get(cache_key)
+        if cached is not None:
+            return cached
+        client = _build_redis_client(opts, decode_responses)
+        _CLIENT_CACHE[cache_key] = client
+        return client
+
+
+def _build_redis_client(opts: RedisOptions, decode_responses: bool) -> redis.Redis:
     # Decide between standard Redis and Sentinel-based Redis
     if opts.is_sentinel and opts.sentinel_hosts:
         logger.debug(
@@ -202,8 +240,39 @@ def create_redis_client(options: Optional[RedisOptions] = None) -> redis.Redis:
     )
 
 
+def close_all_redis_clients() -> None:
+    """Close every cached Redis client and disconnect its pool.
+
+    Intended for clean process shutdown only.  After this call any subsequent
+    create_redis_client() will rebuild fresh clients on demand.
+    """
+    with _CLIENT_CACHE_LOCK:
+        clients = list(_CLIENT_CACHE.values())
+        _CLIENT_CACHE.clear()
+
+    for client in clients:
+        try:
+            pool = getattr(client, "connection_pool", None)
+            if pool is not None:
+                pool.disconnect()
+        except Exception as e:
+            logger.debug(f"[redis-connector] pool disconnect failed: {e}")
+        try:
+            close = getattr(client, "close", None)
+            if callable(close):
+                close()
+        except Exception as e:
+            logger.debug(f"[redis-connector] client close failed: {e}")
+
+
 class RedisConnector:
-    """Creates and holds a Redis client connection based on RedisOptions."""
+    """Returns a process-shared Redis client based on RedisOptions.
+
+    The underlying redis.Redis (and its ConnectionPool) is cached by the
+    effective options inside create_redis_client, so repeated construction
+    of RedisConnector / RedisCatalog / DataWriter / monitoring loggers does
+    not allocate new pools or sockets.
+    """
 
     def __init__(self, options: Optional[RedisOptions] = None):
         self.r = create_redis_client(options)