supertable 2.0.0__tar.gz → 2.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {supertable-2.0.0 → supertable-2.0.1}/PKG-INFO +1 -1
- {supertable-2.0.0 → supertable-2.0.1}/pyproject.toml +1 -1
- {supertable-2.0.0 → supertable-2.0.1}/setup.py +1 -1
- {supertable-2.0.0 → supertable-2.0.1}/supertable/__init__.py +1 -1
- {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/consumers.py +3 -3
- {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/logger.py +1 -1
- {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/reader.py +1 -1
- {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/retention.py +2 -2
- {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/tests/test_retention.py +6 -6
- supertable-2.0.1/supertable/redis_infra.py +467 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable.egg-info/PKG-INFO +1 -1
- {supertable-2.0.0 → supertable-2.0.1}/supertable.egg-info/SOURCES.txt +1 -1
- supertable-2.0.0/supertable/server_common.py +0 -957
- {supertable-2.0.0 → supertable-2.0.1}/LICENSE +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/README.md +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/requirements.txt +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/setup.cfg +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/__init__.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/chain.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/crypto.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/events.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/export.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/middleware.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/tests/__init__.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/tests/test_chain.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/tests/test_crypto.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/tests/test_emit.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/tests/test_events.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/writer_parquet.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/writer_redis.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/config/__init__.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/config/defaults.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/config/homedir.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/config/settings.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/config/tests/__init__.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/config/tests/test_defaults.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/config/tests/test_homedir.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/config/tests/test_settings.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/data_classes.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/data_reader.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/data_writer.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/__init__.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/__init__.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/__main__.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/check_filter_builder.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/controller.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/data_writer_helpers.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/defaults.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/dummy_data.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/read_parquet_header.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s01_01_01_create_super_table.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s01_01_02_enable_mirroring_formats.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s01_02_create_roles.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s01_03_create_users.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s02_01_write_dummy_data.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s02_02_write_single_data.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s02_03_01_write_staging.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s02_03_02_create_pipe.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s02_04_01_write_monitoring_simple.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s02_04_02_write_monitoring_parallel.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s02_05_write_tombstone.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s03_01_read_data_error.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s03_02_01_read_super_data_ok.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s03_02_02_read_table_data_ok.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s03_03_read_meta.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s03_04_read_staging.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s03_06_01_read_roles.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s03_06_02_read_user.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s03_07_01_estimate_read.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s03_07_02_estimate_files.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s03_08_read_snapshot_history.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s04_01_03_delete_pipe.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s05_01_delete_table.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s05_02_delete_super_table.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/webshop/__init__.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/webshop/core.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/webshop/defaults.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/webshop/generate.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/webshop/load.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/webshop/topup.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/__init__.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/data_estimator.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/duckdb_lite.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/duckdb_pro.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/engine_common.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/engine_enum.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/executor.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/plan_stats.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/spark_thrift.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/tests/__init__.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/tests/conftest.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/tests/test_dedup_read.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/tests/test_engine.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/locking/__init__.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/locking/benchmarks/__init__.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/locking/benchmarks/benchmark_locking.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/locking/benchmarks/measure_lock_speed.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/locking/benchmarks/measure_lock_time.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/locking/file_lock.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/locking/redis_lock.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/locking/tests/__init__.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/locking/tests/test_file_lock.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/locking/tests/test_redis_lock.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/logging.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/meta_reader.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/mirroring/__init__.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/mirroring/mirror_delta.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/mirroring/mirror_formats.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/mirroring/mirror_iceberg.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/mirroring/mirror_parquet.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/monitoring_writer.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/plan_extender.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/processing.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/query_plan_manager.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/rbac/__init__.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/rbac/access_control.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/rbac/filter_builder.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/rbac/permissions.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/rbac/role_manager.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/rbac/row_column_security.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/rbac/tests/test_filter_builder.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/rbac/tests/test_rbac.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/rbac/tests/test_rbac_per_table.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/rbac/user_manager.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/redis_catalog.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/redis_connector.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/service_registry.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/simple_table.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/staging_area.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/storage/__init__.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/storage/azure_storage.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/storage/gcp_storage.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/storage/local_storage.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/storage/minio_storage.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/storage/s3_storage.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/storage/storage_factory.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/storage/storage_interface.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/storage/tests/test_storage.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/super_pipe.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/super_table.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/__init__.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_data_reader.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_data_writer.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_data_writer_comprehensive.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_data_writer_tombstones.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_dedup_on_read_write.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_meta_reader.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_newer_than.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_process_delete_only.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_processing.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_query_sql.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_simple_table.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_small_file_compaction.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_super_table.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_supertable_all.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/utils/__init__.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/utils/helper.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/utils/sql_parser.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/utils/tests/test_sql_parser_columns.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable/utils/timer.py +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable.egg-info/dependency_links.txt +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable.egg-info/entry_points.txt +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable.egg-info/requires.txt +0 -0
- {supertable-2.0.0 → supertable-2.0.1}/supertable.egg-info/top_level.txt +0 -0
|
@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
|
|
|
19
19
|
|
|
20
20
|
setup(
|
|
21
21
|
name="supertable",
|
|
22
|
-
version="2.0.
|
|
22
|
+
version="2.0.1",
|
|
23
23
|
description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
|
|
24
24
|
long_description=long_description,
|
|
25
25
|
long_description_content_type="text/markdown",
|
|
@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
|
|
|
25
25
|
project documentation for the full API surface.
|
|
26
26
|
"""
|
|
27
27
|
|
|
28
|
-
__version__ = "2.0.
|
|
28
|
+
__version__ = "2.0.1"
|
|
29
29
|
|
|
30
30
|
# Re-export the core public surface so users can do ``from supertable import …``
|
|
31
31
|
# instead of remembering submodule paths.
|
|
@@ -21,7 +21,7 @@ logger = logging.getLogger(__name__)
|
|
|
21
21
|
def create_consumer(organization: str, group_name: str, start_from: str = "$") -> Dict[str, Any]:
|
|
22
22
|
"""Create an external SIEM consumer group on the audit stream."""
|
|
23
23
|
try:
|
|
24
|
-
from supertable.
|
|
24
|
+
from supertable.redis_infra import redis_client
|
|
25
25
|
from supertable.audit.writer_redis import RedisAuditWriter
|
|
26
26
|
writer = RedisAuditWriter(redis_client, organization, "", maxlen=0)
|
|
27
27
|
ok = writer.create_consumer_group(group_name, start_from)
|
|
@@ -34,7 +34,7 @@ def create_consumer(organization: str, group_name: str, start_from: str = "$") -
|
|
|
34
34
|
def delete_consumer(organization: str, group_name: str) -> Dict[str, Any]:
|
|
35
35
|
"""Remove an external SIEM consumer group."""
|
|
36
36
|
try:
|
|
37
|
-
from supertable.
|
|
37
|
+
from supertable.redis_infra import redis_client
|
|
38
38
|
from supertable.audit.writer_redis import RedisAuditWriter
|
|
39
39
|
writer = RedisAuditWriter(redis_client, organization, "", maxlen=0)
|
|
40
40
|
ok = writer.delete_consumer_group(group_name)
|
|
@@ -47,7 +47,7 @@ def delete_consumer(organization: str, group_name: str) -> Dict[str, Any]:
|
|
|
47
47
|
def list_consumers(organization: str) -> List[Dict[str, Any]]:
|
|
48
48
|
"""List all consumer groups with lag info."""
|
|
49
49
|
try:
|
|
50
|
-
from supertable.
|
|
50
|
+
from supertable.redis_infra import redis_client
|
|
51
51
|
from supertable.audit.writer_redis import RedisAuditWriter
|
|
52
52
|
writer = RedisAuditWriter(redis_client, organization, "", maxlen=0)
|
|
53
53
|
return writer.list_consumer_groups()
|
|
@@ -132,7 +132,7 @@ class AuditLogger:
|
|
|
132
132
|
def _init_redis(self) -> None:
|
|
133
133
|
"""Initialize Redis Stream writer."""
|
|
134
134
|
try:
|
|
135
|
-
from supertable.
|
|
135
|
+
from supertable.redis_infra import redis_client
|
|
136
136
|
from supertable.audit.writer_redis import RedisAuditWriter
|
|
137
137
|
self._redis_writer = RedisAuditWriter(
|
|
138
138
|
redis_client=redis_client,
|
|
@@ -93,7 +93,7 @@ def _query_redis(
|
|
|
93
93
|
) -> List[Dict[str, Any]]:
|
|
94
94
|
"""Query recent events from the Redis Stream."""
|
|
95
95
|
try:
|
|
96
|
-
from supertable.
|
|
96
|
+
from supertable.redis_infra import redis_client
|
|
97
97
|
from supertable.audit.writer_redis import RedisAuditWriter
|
|
98
98
|
writer = RedisAuditWriter(redis_client, organization, "", maxlen=0)
|
|
99
99
|
return writer.query(start_ms=start_ms, end_ms=end_ms, count=limit)
|
|
@@ -86,7 +86,7 @@ def is_legal_hold_active(organization: str) -> bool:
|
|
|
86
86
|
"""
|
|
87
87
|
# Try Redis first (runtime override)
|
|
88
88
|
try:
|
|
89
|
-
from supertable.
|
|
89
|
+
from supertable.redis_infra import redis_client
|
|
90
90
|
val = redis_client.get(_legal_hold_key(organization))
|
|
91
91
|
if val is not None:
|
|
92
92
|
decoded = val if isinstance(val, str) else val.decode("utf-8")
|
|
@@ -128,7 +128,7 @@ def set_legal_hold(enabled: bool, organization: str = "") -> Dict[str, Any]:
|
|
|
128
128
|
|
|
129
129
|
# Persist to Redis
|
|
130
130
|
try:
|
|
131
|
-
from supertable.
|
|
131
|
+
from supertable.redis_infra import redis_client
|
|
132
132
|
redis_client.set(_legal_hold_key(organization), "1" if enabled else "0")
|
|
133
133
|
except Exception as e:
|
|
134
134
|
logger.error("[audit-retention] Failed to persist legal hold to Redis: %s", e)
|
|
@@ -83,7 +83,7 @@ class TestIsLegalHoldActive:
|
|
|
83
83
|
) -> None:
|
|
84
84
|
fake_client = _make_redis_client(get_value=redis_value)
|
|
85
85
|
fake_module = SimpleNamespace(redis_client=fake_client)
|
|
86
|
-
monkeypatch.setitem(__import__("sys").modules, "supertable.
|
|
86
|
+
monkeypatch.setitem(__import__("sys").modules, "supertable.redis_infra", fake_module)
|
|
87
87
|
assert retention.is_legal_hold_active("acme") is True
|
|
88
88
|
|
|
89
89
|
@pytest.mark.parametrize("redis_value", [b"0", "false", b"no"])
|
|
@@ -92,7 +92,7 @@ class TestIsLegalHoldActive:
|
|
|
92
92
|
) -> None:
|
|
93
93
|
fake_client = _make_redis_client(get_value=redis_value)
|
|
94
94
|
fake_module = SimpleNamespace(redis_client=fake_client)
|
|
95
|
-
monkeypatch.setitem(__import__("sys").modules, "supertable.
|
|
95
|
+
monkeypatch.setitem(__import__("sys").modules, "supertable.redis_infra", fake_module)
|
|
96
96
|
assert retention.is_legal_hold_active("acme") is False
|
|
97
97
|
|
|
98
98
|
def test_falls_back_to_settings_when_redis_returns_none(
|
|
@@ -102,7 +102,7 @@ class TestIsLegalHoldActive:
|
|
|
102
102
|
fake_server_common = SimpleNamespace(redis_client=fake_client)
|
|
103
103
|
monkeypatch.setitem(
|
|
104
104
|
__import__("sys").modules,
|
|
105
|
-
"supertable.
|
|
105
|
+
"supertable.redis_infra",
|
|
106
106
|
fake_server_common,
|
|
107
107
|
)
|
|
108
108
|
|
|
@@ -125,7 +125,7 @@ class TestIsLegalHoldActive:
|
|
|
125
125
|
raise RuntimeError("redis unreachable")
|
|
126
126
|
|
|
127
127
|
monkeypatch.setitem(
|
|
128
|
-
__import__("sys").modules, "supertable.
|
|
128
|
+
__import__("sys").modules, "supertable.redis_infra", BoomModule()
|
|
129
129
|
)
|
|
130
130
|
|
|
131
131
|
# Make settings import fail by removing the attribute
|
|
@@ -155,7 +155,7 @@ class TestSetLegalHold:
|
|
|
155
155
|
) -> None:
|
|
156
156
|
fake_client = _make_redis_client()
|
|
157
157
|
fake_module = SimpleNamespace(redis_client=fake_client)
|
|
158
|
-
monkeypatch.setitem(__import__("sys").modules, "supertable.
|
|
158
|
+
monkeypatch.setitem(__import__("sys").modules, "supertable.redis_infra", fake_module)
|
|
159
159
|
|
|
160
160
|
# Stub out the audit emit chain so the test never touches the real logger.
|
|
161
161
|
import supertable.audit as audit_pkg
|
|
@@ -181,7 +181,7 @@ class TestSetLegalHold:
|
|
|
181
181
|
fake_client = MagicMock()
|
|
182
182
|
fake_client.set.side_effect = RuntimeError("redis down")
|
|
183
183
|
fake_module = SimpleNamespace(redis_client=fake_client)
|
|
184
|
-
monkeypatch.setitem(__import__("sys").modules, "supertable.
|
|
184
|
+
monkeypatch.setitem(__import__("sys").modules, "supertable.redis_infra", fake_module)
|
|
185
185
|
|
|
186
186
|
result = retention.set_legal_hold(False, organization="acme")
|
|
187
187
|
assert result["ok"] is False
|
|
@@ -0,0 +1,467 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import json
|
|
5
|
+
from typing import Dict, Iterator, List, Optional, Tuple
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from urllib.parse import urlparse
|
|
8
|
+
import time
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
import redis
|
|
12
|
+
from redis import Sentinel
|
|
13
|
+
from redis.sentinel import MasterNotFoundError
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
from supertable.config.settings import settings as _cfg
|
|
18
|
+
|
|
19
|
+
logger = logging.getLogger(__name__)
|
|
20
|
+
# ------------------------------ Settings ------------------------------
|
|
21
|
+
|
|
22
|
+
class Settings:
|
|
23
|
+
"""Thin adapter that delegates to the central config.settings singleton.
|
|
24
|
+
|
|
25
|
+
Attributes that config.settings does not carry (e.g. TEMPLATES_DIR with
|
|
26
|
+
a path derived from __file__) are resolved here.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
def __init__(self) -> None:
|
|
30
|
+
self.SUPERTABLE_ORGANIZATION: str = _cfg.SUPERTABLE_ORGANIZATION
|
|
31
|
+
self.SUPERTABLE_SUPERUSER_TOKEN: str = _cfg.SUPERTABLE_SUPERUSER_TOKEN
|
|
32
|
+
self.SUPERTABLE_SESSION_SECRET: str = _cfg.SUPERTABLE_SESSION_SECRET
|
|
33
|
+
|
|
34
|
+
self.SUPERTABLE_REDIS_URL: Optional[str] = _cfg.SUPERTABLE_REDIS_URL or None
|
|
35
|
+
self.SUPERTABLE_REDIS_HOST: str = _cfg.SUPERTABLE_REDIS_HOST
|
|
36
|
+
self.SUPERTABLE_REDIS_PORT: int = _cfg.SUPERTABLE_REDIS_PORT
|
|
37
|
+
self.SUPERTABLE_REDIS_DB: int = _cfg.SUPERTABLE_REDIS_DB
|
|
38
|
+
self.SUPERTABLE_REDIS_PASSWORD: Optional[str] = _cfg.SUPERTABLE_REDIS_PASSWORD or None
|
|
39
|
+
self.SUPERTABLE_REDIS_USERNAME: Optional[str] = _cfg.SUPERTABLE_REDIS_USERNAME or None
|
|
40
|
+
|
|
41
|
+
self.SUPERTABLE_REDIS_SENTINEL: Optional[str] = str(_cfg.SUPERTABLE_REDIS_SENTINEL) if _cfg.SUPERTABLE_REDIS_SENTINEL else None
|
|
42
|
+
self.SUPERTABLE_REDIS_SENTINELS: Optional[str] = _cfg.SUPERTABLE_REDIS_SENTINELS or None
|
|
43
|
+
self.SUPERTABLE_REDIS_SENTINEL_MASTER: Optional[str] = _cfg.SUPERTABLE_REDIS_SENTINEL_MASTER or None
|
|
44
|
+
self.SUPERTABLE_REDIS_SENTINEL_PASSWORD: Optional[str] = _cfg.SUPERTABLE_REDIS_SENTINEL_PASSWORD or None
|
|
45
|
+
self.SUPERTABLE_REDIS_SENTINEL_STRICT: Optional[str] = _cfg.SUPERTABLE_REDIS_SENTINEL_STRICT or None
|
|
46
|
+
|
|
47
|
+
self.SUPERTABLE_LOGIN_MASK: int = _cfg.SUPERTABLE_LOGIN_MASK
|
|
48
|
+
|
|
49
|
+
self.DOTENV_PATH: str = _cfg.DOTENV_PATH
|
|
50
|
+
|
|
51
|
+
# TEMPLATES_DIR: use central setting if provided, otherwise derive from __file__
|
|
52
|
+
self.TEMPLATES_DIR: str = (
|
|
53
|
+
_cfg.TEMPLATES_DIR
|
|
54
|
+
or str(Path(__file__).resolve().parent / "webui" / "templates")
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
self.SECURE_COOKIES: bool = _cfg.SECURE_COOKIES
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
settings = Settings()
|
|
61
|
+
if settings.SUPERTABLE_LOGIN_MASK not in (1, 2, 3):
|
|
62
|
+
raise RuntimeError(
|
|
63
|
+
f"Invalid SUPERTABLE_LOGIN_MASK (must be 1, 2, or 3): {settings.SUPERTABLE_LOGIN_MASK}"
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
_missing_envs: List[str] = []
|
|
67
|
+
if not settings.SUPERTABLE_ORGANIZATION:
|
|
68
|
+
_missing_envs.append("SUPERTABLE_ORGANIZATION")
|
|
69
|
+
if not (settings.SUPERTABLE_SUPERUSER_TOKEN or "").strip():
|
|
70
|
+
_missing_envs.append("SUPERTABLE_SUPERUSER_TOKEN")
|
|
71
|
+
if _missing_envs:
|
|
72
|
+
raise RuntimeError("Missing required environment variables: " + ", ".join(_missing_envs))
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _now_ms() -> int:
|
|
76
|
+
from time import time as _t
|
|
77
|
+
return int(_t() * 1000)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
# ------------------------------ Catalog (import or fallback) ------------------------------
|
|
81
|
+
|
|
82
|
+
def _root_key(org: str, sup: str) -> str:
|
|
83
|
+
return f"supertable:{org}:{sup}:meta:root"
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _leaf_key(org: str, sup: str, simple: str) -> str:
|
|
87
|
+
return f"supertable:{org}:{sup}:meta:leaf:{simple}"
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _mirrors_key(org: str, sup: str) -> str:
|
|
91
|
+
return f"supertable:{org}:{sup}:meta:mirrors"
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class _FallbackCatalog:
|
|
95
|
+
def __init__(self, r: redis.Redis):
|
|
96
|
+
self.r = r
|
|
97
|
+
|
|
98
|
+
def ensure_root(self, org: str, sup: str) -> None:
|
|
99
|
+
key = _root_key(org, sup)
|
|
100
|
+
if not self.r.exists(key):
|
|
101
|
+
self.r.set(key, json.dumps({"version": 0, "ts": _now_ms()}))
|
|
102
|
+
|
|
103
|
+
def get_root(self, org: str, sup: str) -> Optional[Dict]:
|
|
104
|
+
raw = self.r.get(_root_key(org, sup))
|
|
105
|
+
return json.loads(raw) if raw else None
|
|
106
|
+
|
|
107
|
+
def get_leaf(self, org: str, sup: str, simple: str) -> Optional[Dict]:
|
|
108
|
+
raw = self.r.get(_leaf_key(org, sup, simple))
|
|
109
|
+
return json.loads(raw) if raw else None
|
|
110
|
+
|
|
111
|
+
def get_mirrors(self, org: str, sup: str) -> List[str]:
|
|
112
|
+
raw = self.r.get(_mirrors_key(org, sup))
|
|
113
|
+
if not raw:
|
|
114
|
+
return []
|
|
115
|
+
try:
|
|
116
|
+
obj = json.loads(raw)
|
|
117
|
+
except Exception:
|
|
118
|
+
return []
|
|
119
|
+
out = []
|
|
120
|
+
for f in (obj.get("formats") or []):
|
|
121
|
+
fu = str(f).upper()
|
|
122
|
+
if fu in ("DELTA", "ICEBERG", "PARQUET") and fu not in out:
|
|
123
|
+
out.append(fu)
|
|
124
|
+
return out
|
|
125
|
+
|
|
126
|
+
def set_mirrors(self, org: str, sup: str, formats: List[str]) -> List[str]:
|
|
127
|
+
uniq = []
|
|
128
|
+
for f in (formats or []):
|
|
129
|
+
fu = str(f).upper()
|
|
130
|
+
if fu in ("DELTA", "ICEBERG", "PARQUET") and fu not in uniq:
|
|
131
|
+
uniq.append(fu)
|
|
132
|
+
self.r.set(_mirrors_key(org, sup), json.dumps({"formats": uniq, "ts": _now_ms()}))
|
|
133
|
+
return uniq
|
|
134
|
+
|
|
135
|
+
def enable_mirror(self, org: str, sup: str, fmt: str) -> List[str]:
|
|
136
|
+
cur = self.get_mirrors(org, sup)
|
|
137
|
+
fu = str(fmt).upper()
|
|
138
|
+
if fu not in ("DELTA", "ICEBERG", "PARQUET") or fu in cur:
|
|
139
|
+
return cur
|
|
140
|
+
return self.set_mirrors(org, sup, cur + [fu])
|
|
141
|
+
|
|
142
|
+
def disable_mirror(self, org: str, sup: str, fmt: str) -> List[str]:
|
|
143
|
+
cur = self.get_mirrors(org, sup)
|
|
144
|
+
fu = str(fmt).upper()
|
|
145
|
+
nxt = [x for x in cur if x != fu]
|
|
146
|
+
return self.set_mirrors(org, sup, nxt)
|
|
147
|
+
|
|
148
|
+
def scan_leaf_keys(self, org: str, sup: str, count: int = 1000) -> Iterator[str]:
|
|
149
|
+
pattern = f"supertable:{org}:{sup}:meta:leaf:*"
|
|
150
|
+
cursor = 0
|
|
151
|
+
while True:
|
|
152
|
+
cursor, keys = self.r.scan(cursor=cursor, match=pattern, count=max(1, int(count)))
|
|
153
|
+
for k in keys:
|
|
154
|
+
yield k if isinstance(k, str) else k.decode("utf-8")
|
|
155
|
+
if cursor == 0:
|
|
156
|
+
break
|
|
157
|
+
|
|
158
|
+
def scan_leaf_items(self, org: str, sup: str, count: int = 1000) -> Iterator[Dict]:
|
|
159
|
+
batch: List[str] = []
|
|
160
|
+
for key in self.scan_leaf_keys(org, sup, count=count):
|
|
161
|
+
batch.append(key)
|
|
162
|
+
if len(batch) >= count:
|
|
163
|
+
yield from self._fetch_batch(batch)
|
|
164
|
+
batch = []
|
|
165
|
+
if batch:
|
|
166
|
+
yield from self._fetch_batch(batch)
|
|
167
|
+
|
|
168
|
+
def _fetch_batch(self, keys: List[str]) -> Iterator[Dict]:
|
|
169
|
+
pipe = self.r.pipeline()
|
|
170
|
+
for k in keys:
|
|
171
|
+
pipe.get(k)
|
|
172
|
+
vals = pipe.execute()
|
|
173
|
+
for k, raw in zip(keys, vals):
|
|
174
|
+
if not raw:
|
|
175
|
+
continue
|
|
176
|
+
try:
|
|
177
|
+
obj = json.loads(raw if isinstance(raw, str) else raw.decode("utf-8"))
|
|
178
|
+
simple = k.rsplit("meta:leaf:", 1)[-1]
|
|
179
|
+
yield {
|
|
180
|
+
"simple": simple,
|
|
181
|
+
"version": int(obj.get("version", -1)),
|
|
182
|
+
"ts": int(obj.get("ts", 0)),
|
|
183
|
+
"path": obj.get("path", ""),
|
|
184
|
+
}
|
|
185
|
+
except Exception:
|
|
186
|
+
continue
|
|
187
|
+
|
|
188
|
+
# -- RBAC methods (mirrors RedisCatalog API using correct rbac: key namespace) --
|
|
189
|
+
|
|
190
|
+
@staticmethod
|
|
191
|
+
def _decode_member(m) -> str:
|
|
192
|
+
return m if isinstance(m, str) else m.decode("utf-8")
|
|
193
|
+
|
|
194
|
+
def get_users(self, org: str, sup: str) -> List[Dict]:
|
|
195
|
+
users: List[Dict] = []
|
|
196
|
+
try:
|
|
197
|
+
index_key = f"supertable:{org}:{sup}:rbac:users:index"
|
|
198
|
+
members = self.r.smembers(index_key)
|
|
199
|
+
for uid_raw in (members or []):
|
|
200
|
+
uid = self._decode_member(uid_raw)
|
|
201
|
+
doc_key = f"supertable:{org}:{sup}:rbac:users:doc:{uid}"
|
|
202
|
+
raw = self.r.hgetall(doc_key)
|
|
203
|
+
if raw:
|
|
204
|
+
data: Dict = dict(raw)
|
|
205
|
+
data.setdefault("user_id", uid)
|
|
206
|
+
data.setdefault("hash", uid)
|
|
207
|
+
if "roles" in data:
|
|
208
|
+
try:
|
|
209
|
+
data["roles"] = json.loads(data["roles"])
|
|
210
|
+
except (json.JSONDecodeError, TypeError):
|
|
211
|
+
data["roles"] = []
|
|
212
|
+
users.append(data)
|
|
213
|
+
except Exception as e:
|
|
214
|
+
logger.warning("_FallbackCatalog.get_users error: %s", e)
|
|
215
|
+
return users
|
|
216
|
+
|
|
217
|
+
def get_roles(self, org: str, sup: str) -> List[Dict]:
|
|
218
|
+
roles: List[Dict] = []
|
|
219
|
+
try:
|
|
220
|
+
index_key = f"supertable:{org}:{sup}:rbac:roles:index"
|
|
221
|
+
members = self.r.smembers(index_key)
|
|
222
|
+
for rid_raw in (members or []):
|
|
223
|
+
rid = self._decode_member(rid_raw)
|
|
224
|
+
doc_key = f"supertable:{org}:{sup}:rbac:roles:doc:{rid}"
|
|
225
|
+
raw = self.r.hgetall(doc_key)
|
|
226
|
+
if raw:
|
|
227
|
+
data: Dict = dict(raw)
|
|
228
|
+
data.setdefault("role_id", rid)
|
|
229
|
+
data.setdefault("hash", rid)
|
|
230
|
+
for field in ("tables", "columns", "filters"):
|
|
231
|
+
if field in data:
|
|
232
|
+
try:
|
|
233
|
+
data[field] = json.loads(data[field])
|
|
234
|
+
except (json.JSONDecodeError, TypeError):
|
|
235
|
+
pass
|
|
236
|
+
roles.append(data)
|
|
237
|
+
except Exception as e:
|
|
238
|
+
logger.warning("_FallbackCatalog.get_roles error: %s", e)
|
|
239
|
+
return roles
|
|
240
|
+
|
|
241
|
+
def get_user_details(self, org: str, sup: str, user_id: str) -> Optional[Dict]:
|
|
242
|
+
try:
|
|
243
|
+
doc_key = f"supertable:{org}:{sup}:rbac:users:doc:{user_id}"
|
|
244
|
+
raw = self.r.hgetall(doc_key)
|
|
245
|
+
if not raw:
|
|
246
|
+
return None
|
|
247
|
+
data: Dict = dict(raw)
|
|
248
|
+
if "roles" in data:
|
|
249
|
+
try:
|
|
250
|
+
data["roles"] = json.loads(data["roles"])
|
|
251
|
+
except (json.JSONDecodeError, TypeError):
|
|
252
|
+
data["roles"] = []
|
|
253
|
+
return data
|
|
254
|
+
except Exception as e:
|
|
255
|
+
logger.warning("_FallbackCatalog.get_user_details error: %s", e)
|
|
256
|
+
return None
|
|
257
|
+
|
|
258
|
+
def get_role_details(self, org: str, sup: str, role_id: str) -> Optional[Dict]:
|
|
259
|
+
try:
|
|
260
|
+
doc_key = f"supertable:{org}:{sup}:rbac:roles:doc:{role_id}"
|
|
261
|
+
raw = self.r.hgetall(doc_key)
|
|
262
|
+
if not raw:
|
|
263
|
+
return None
|
|
264
|
+
data: Dict = dict(raw)
|
|
265
|
+
for field in ("tables", "columns", "filters"):
|
|
266
|
+
if field in data:
|
|
267
|
+
try:
|
|
268
|
+
data[field] = json.loads(data[field])
|
|
269
|
+
except (json.JSONDecodeError, TypeError):
|
|
270
|
+
pass
|
|
271
|
+
return data
|
|
272
|
+
except Exception as e:
|
|
273
|
+
logger.warning("_FallbackCatalog.get_role_details error: %s", e)
|
|
274
|
+
return None
|
|
275
|
+
|
|
276
|
+
def rbac_get_user_id_by_username(self, org: str, sup: str, username: str) -> Optional[str]:
|
|
277
|
+
try:
|
|
278
|
+
name_map_key = f"supertable:{org}:{sup}:rbac:users:name_to_id"
|
|
279
|
+
val = self.r.hget(name_map_key, username.lower())
|
|
280
|
+
if val is None:
|
|
281
|
+
return None
|
|
282
|
+
return self._decode_member(val)
|
|
283
|
+
except Exception as e:
|
|
284
|
+
logger.warning("_FallbackCatalog.rbac_get_user_id_by_username error: %s", e)
|
|
285
|
+
return None
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _coerce_password(pw: Optional[str]) -> Optional[str]:
|
|
289
|
+
if pw is None:
|
|
290
|
+
return None
|
|
291
|
+
v = pw.strip()
|
|
292
|
+
# Treat these as "no password"
|
|
293
|
+
if v in ("", "None", "none", "null", "NULL"):
|
|
294
|
+
return None
|
|
295
|
+
return v
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _build_redis_client() -> redis.Redis:
|
|
299
|
+
"""
|
|
300
|
+
Build a Redis client from SUPERTABLE_* envs.
|
|
301
|
+
|
|
302
|
+
Precedence:
|
|
303
|
+
1) SUPERTABLE_REDIS_URL (parsed)
|
|
304
|
+
2) SUPERTABLE_REDIS_HOST/PORT/DB/PASSWORD (overrides URL parts if provided)
|
|
305
|
+
|
|
306
|
+
Sentinel:
|
|
307
|
+
If SUPERTABLE_REDIS_SENTINEL is enabled and SUPERTABLE_REDIS_SENTINELS is set,
|
|
308
|
+
use the same Sentinel connection behavior as RedisCatalog (ping-probe + optional fallback).
|
|
309
|
+
"""
|
|
310
|
+
settings = Settings()
|
|
311
|
+
url = (settings.SUPERTABLE_REDIS_URL or "").strip() or None
|
|
312
|
+
|
|
313
|
+
host = settings.SUPERTABLE_REDIS_HOST
|
|
314
|
+
port = settings.SUPERTABLE_REDIS_PORT
|
|
315
|
+
db = settings.SUPERTABLE_REDIS_DB
|
|
316
|
+
username = (settings.SUPERTABLE_REDIS_USERNAME or "").strip() or None
|
|
317
|
+
password = _coerce_password(settings.SUPERTABLE_REDIS_PASSWORD)
|
|
318
|
+
|
|
319
|
+
use_ssl = url.startswith("rediss://") if url else False
|
|
320
|
+
|
|
321
|
+
if url:
|
|
322
|
+
u = urlparse(url)
|
|
323
|
+
if u.scheme not in ("redis", "rediss"):
|
|
324
|
+
raise ValueError(f"Unsupported Redis URL scheme: {u.scheme}")
|
|
325
|
+
# Extract from URL
|
|
326
|
+
if u.hostname:
|
|
327
|
+
host = u.hostname
|
|
328
|
+
if u.port:
|
|
329
|
+
port = u.port
|
|
330
|
+
# db from path: "/0", "/1", ...
|
|
331
|
+
if u.path and len(u.path) > 1:
|
|
332
|
+
try:
|
|
333
|
+
db_from_url = int(u.path.lstrip("/"))
|
|
334
|
+
db = db_from_url
|
|
335
|
+
except Exception:
|
|
336
|
+
pass
|
|
337
|
+
if u.username:
|
|
338
|
+
username = u.username
|
|
339
|
+
if u.password:
|
|
340
|
+
password = _coerce_password(u.password)
|
|
341
|
+
|
|
342
|
+
# Sentinel detection + options
|
|
343
|
+
sentinel_enabled = (settings.SUPERTABLE_REDIS_SENTINEL or "").strip().lower() in (
|
|
344
|
+
"1",
|
|
345
|
+
"true",
|
|
346
|
+
"yes",
|
|
347
|
+
"y",
|
|
348
|
+
"on",
|
|
349
|
+
)
|
|
350
|
+
sentinel_strict = (settings.SUPERTABLE_REDIS_SENTINEL_STRICT or "").strip().lower() in (
|
|
351
|
+
"1",
|
|
352
|
+
"true",
|
|
353
|
+
"yes",
|
|
354
|
+
"y",
|
|
355
|
+
"on",
|
|
356
|
+
)
|
|
357
|
+
sentinel_master = (settings.SUPERTABLE_REDIS_SENTINEL_MASTER or "").strip() or "mymaster"
|
|
358
|
+
|
|
359
|
+
sentinel_password = _coerce_password(settings.SUPERTABLE_REDIS_SENTINEL_PASSWORD) or password
|
|
360
|
+
|
|
361
|
+
# Single-password setup: if Sentinel is enabled and Redis password is not set, reuse Sentinel password.
|
|
362
|
+
if sentinel_enabled and password is None and sentinel_password:
|
|
363
|
+
password = sentinel_password
|
|
364
|
+
|
|
365
|
+
# If username is set but password is None, drop username (ACL requires both)
|
|
366
|
+
if username and not password:
|
|
367
|
+
username = None
|
|
368
|
+
|
|
369
|
+
sentinel_hosts: List[Tuple[str, int]] = []
|
|
370
|
+
sentinel_raw = (settings.SUPERTABLE_REDIS_SENTINELS or "").strip()
|
|
371
|
+
if sentinel_raw:
|
|
372
|
+
for part in sentinel_raw.split(","):
|
|
373
|
+
part = part.strip()
|
|
374
|
+
if not part:
|
|
375
|
+
continue
|
|
376
|
+
try:
|
|
377
|
+
h, p = part.split(":")
|
|
378
|
+
sentinel_hosts.append((h.strip(), int(p)))
|
|
379
|
+
except ValueError:
|
|
380
|
+
# Keep behavior non-fatal; invalid entries are ignored.
|
|
381
|
+
continue
|
|
382
|
+
|
|
383
|
+
if sentinel_enabled and sentinel_hosts:
|
|
384
|
+
sentinel_kwargs: dict = {
|
|
385
|
+
"socket_timeout": 0.5,
|
|
386
|
+
"decode_responses": True,
|
|
387
|
+
"ssl": use_ssl,
|
|
388
|
+
}
|
|
389
|
+
if sentinel_password:
|
|
390
|
+
sentinel_kwargs["password"] = sentinel_password
|
|
391
|
+
if username:
|
|
392
|
+
sentinel_kwargs["username"] = username
|
|
393
|
+
|
|
394
|
+
sentinel = Sentinel(
|
|
395
|
+
sentinel_hosts,
|
|
396
|
+
sentinel_kwargs=sentinel_kwargs,
|
|
397
|
+
socket_timeout=0.5,
|
|
398
|
+
decode_responses=True,
|
|
399
|
+
ssl=use_ssl,
|
|
400
|
+
username=username,
|
|
401
|
+
password=password,
|
|
402
|
+
)
|
|
403
|
+
|
|
404
|
+
sentinel_client: redis.Redis = sentinel.master_for(
|
|
405
|
+
sentinel_master,
|
|
406
|
+
db=db,
|
|
407
|
+
decode_responses=True,
|
|
408
|
+
ssl=use_ssl,
|
|
409
|
+
username=username,
|
|
410
|
+
password=password,
|
|
411
|
+
)
|
|
412
|
+
|
|
413
|
+
# Fail-fast probe (matches RedisCatalog behavior).
|
|
414
|
+
sentinel_err: Optional[BaseException] = None
|
|
415
|
+
deadline = time.time() + 3.0
|
|
416
|
+
while time.time() < deadline:
|
|
417
|
+
try:
|
|
418
|
+
sentinel_client.ping()
|
|
419
|
+
sentinel_err = None
|
|
420
|
+
break
|
|
421
|
+
except (MasterNotFoundError, redis.RedisError, OSError) as e:
|
|
422
|
+
sentinel_err = e
|
|
423
|
+
time.sleep(0.2)
|
|
424
|
+
|
|
425
|
+
if sentinel_err is None:
|
|
426
|
+
return sentinel_client
|
|
427
|
+
|
|
428
|
+
if sentinel_strict:
|
|
429
|
+
raise sentinel_err
|
|
430
|
+
|
|
431
|
+
# Non-strict fallback to standard Redis
|
|
432
|
+
return redis.Redis(
|
|
433
|
+
host=host,
|
|
434
|
+
port=port,
|
|
435
|
+
db=db,
|
|
436
|
+
password=password,
|
|
437
|
+
username=username,
|
|
438
|
+
decode_responses=True,
|
|
439
|
+
ssl=use_ssl,
|
|
440
|
+
)
|
|
441
|
+
|
|
442
|
+
# Standard Redis mode
|
|
443
|
+
return redis.Redis(
|
|
444
|
+
host=host,
|
|
445
|
+
port=port,
|
|
446
|
+
db=db,
|
|
447
|
+
password=password,
|
|
448
|
+
username=username,
|
|
449
|
+
decode_responses=True,
|
|
450
|
+
ssl=use_ssl,
|
|
451
|
+
)
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _build_catalog() -> Tuple[object, redis.Redis]:
|
|
455
|
+
r = _build_redis_client()
|
|
456
|
+
try:
|
|
457
|
+
from supertable.redis_catalog import RedisCatalog as _RC # type: ignore
|
|
458
|
+
return _RC(), r
|
|
459
|
+
except Exception:
|
|
460
|
+
try:
|
|
461
|
+
from redis_catalog import RedisCatalog as _RC # type: ignore
|
|
462
|
+
return _RC(), r
|
|
463
|
+
except Exception:
|
|
464
|
+
return _FallbackCatalog(r), r
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
catalog, redis_client = _build_catalog()
|
|
@@ -15,7 +15,7 @@ supertable/processing.py
|
|
|
15
15
|
supertable/query_plan_manager.py
|
|
16
16
|
supertable/redis_catalog.py
|
|
17
17
|
supertable/redis_connector.py
|
|
18
|
-
supertable/
|
|
18
|
+
supertable/redis_infra.py
|
|
19
19
|
supertable/service_registry.py
|
|
20
20
|
supertable/simple_table.py
|
|
21
21
|
supertable/staging_area.py
|