supertable 2.0.0__tar.gz → 2.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. {supertable-2.0.0 → supertable-2.0.1}/PKG-INFO +1 -1
  2. {supertable-2.0.0 → supertable-2.0.1}/pyproject.toml +1 -1
  3. {supertable-2.0.0 → supertable-2.0.1}/setup.py +1 -1
  4. {supertable-2.0.0 → supertable-2.0.1}/supertable/__init__.py +1 -1
  5. {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/consumers.py +3 -3
  6. {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/logger.py +1 -1
  7. {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/reader.py +1 -1
  8. {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/retention.py +2 -2
  9. {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/tests/test_retention.py +6 -6
  10. supertable-2.0.1/supertable/redis_infra.py +467 -0
  11. {supertable-2.0.0 → supertable-2.0.1}/supertable.egg-info/PKG-INFO +1 -1
  12. {supertable-2.0.0 → supertable-2.0.1}/supertable.egg-info/SOURCES.txt +1 -1
  13. supertable-2.0.0/supertable/server_common.py +0 -957
  14. {supertable-2.0.0 → supertable-2.0.1}/LICENSE +0 -0
  15. {supertable-2.0.0 → supertable-2.0.1}/README.md +0 -0
  16. {supertable-2.0.0 → supertable-2.0.1}/requirements.txt +0 -0
  17. {supertable-2.0.0 → supertable-2.0.1}/setup.cfg +0 -0
  18. {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/__init__.py +0 -0
  19. {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/chain.py +0 -0
  20. {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/crypto.py +0 -0
  21. {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/events.py +0 -0
  22. {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/export.py +0 -0
  23. {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/middleware.py +0 -0
  24. {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/tests/__init__.py +0 -0
  25. {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/tests/test_chain.py +0 -0
  26. {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/tests/test_crypto.py +0 -0
  27. {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/tests/test_emit.py +0 -0
  28. {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/tests/test_events.py +0 -0
  29. {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/writer_parquet.py +0 -0
  30. {supertable-2.0.0 → supertable-2.0.1}/supertable/audit/writer_redis.py +0 -0
  31. {supertable-2.0.0 → supertable-2.0.1}/supertable/config/__init__.py +0 -0
  32. {supertable-2.0.0 → supertable-2.0.1}/supertable/config/defaults.py +0 -0
  33. {supertable-2.0.0 → supertable-2.0.1}/supertable/config/homedir.py +0 -0
  34. {supertable-2.0.0 → supertable-2.0.1}/supertable/config/settings.py +0 -0
  35. {supertable-2.0.0 → supertable-2.0.1}/supertable/config/tests/__init__.py +0 -0
  36. {supertable-2.0.0 → supertable-2.0.1}/supertable/config/tests/test_defaults.py +0 -0
  37. {supertable-2.0.0 → supertable-2.0.1}/supertable/config/tests/test_homedir.py +0 -0
  38. {supertable-2.0.0 → supertable-2.0.1}/supertable/config/tests/test_settings.py +0 -0
  39. {supertable-2.0.0 → supertable-2.0.1}/supertable/data_classes.py +0 -0
  40. {supertable-2.0.0 → supertable-2.0.1}/supertable/data_reader.py +0 -0
  41. {supertable-2.0.0 → supertable-2.0.1}/supertable/data_writer.py +0 -0
  42. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/__init__.py +0 -0
  43. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/__init__.py +0 -0
  44. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/__main__.py +0 -0
  45. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/check_filter_builder.py +0 -0
  46. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/controller.py +0 -0
  47. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/data_writer_helpers.py +0 -0
  48. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/defaults.py +0 -0
  49. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/dummy_data.py +0 -0
  50. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/read_parquet_header.py +0 -0
  51. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s01_01_01_create_super_table.py +0 -0
  52. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s01_01_02_enable_mirroring_formats.py +0 -0
  53. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s01_02_create_roles.py +0 -0
  54. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s01_03_create_users.py +0 -0
  55. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s02_01_write_dummy_data.py +0 -0
  56. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s02_02_write_single_data.py +0 -0
  57. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s02_03_01_write_staging.py +0 -0
  58. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s02_03_02_create_pipe.py +0 -0
  59. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s02_04_01_write_monitoring_simple.py +0 -0
  60. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s02_04_02_write_monitoring_parallel.py +0 -0
  61. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s02_05_write_tombstone.py +0 -0
  62. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s03_01_read_data_error.py +0 -0
  63. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s03_02_01_read_super_data_ok.py +0 -0
  64. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s03_02_02_read_table_data_ok.py +0 -0
  65. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s03_03_read_meta.py +0 -0
  66. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s03_04_read_staging.py +0 -0
  67. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s03_06_01_read_roles.py +0 -0
  68. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s03_06_02_read_user.py +0 -0
  69. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s03_07_01_estimate_read.py +0 -0
  70. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s03_07_02_estimate_files.py +0 -0
  71. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s03_08_read_snapshot_history.py +0 -0
  72. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s04_01_03_delete_pipe.py +0 -0
  73. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s05_01_delete_table.py +0 -0
  74. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/quickstart/s05_02_delete_super_table.py +0 -0
  75. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/webshop/__init__.py +0 -0
  76. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/webshop/core.py +0 -0
  77. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/webshop/defaults.py +0 -0
  78. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/webshop/generate.py +0 -0
  79. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/webshop/load.py +0 -0
  80. {supertable-2.0.0 → supertable-2.0.1}/supertable/demo/webshop/topup.py +0 -0
  81. {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/__init__.py +0 -0
  82. {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/data_estimator.py +0 -0
  83. {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/duckdb_lite.py +0 -0
  84. {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/duckdb_pro.py +0 -0
  85. {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/engine_common.py +0 -0
  86. {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/engine_enum.py +0 -0
  87. {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/executor.py +0 -0
  88. {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/plan_stats.py +0 -0
  89. {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/spark_thrift.py +0 -0
  90. {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/tests/__init__.py +0 -0
  91. {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/tests/conftest.py +0 -0
  92. {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/tests/test_dedup_read.py +0 -0
  93. {supertable-2.0.0 → supertable-2.0.1}/supertable/engine/tests/test_engine.py +0 -0
  94. {supertable-2.0.0 → supertable-2.0.1}/supertable/locking/__init__.py +0 -0
  95. {supertable-2.0.0 → supertable-2.0.1}/supertable/locking/benchmarks/__init__.py +0 -0
  96. {supertable-2.0.0 → supertable-2.0.1}/supertable/locking/benchmarks/benchmark_locking.py +0 -0
  97. {supertable-2.0.0 → supertable-2.0.1}/supertable/locking/benchmarks/measure_lock_speed.py +0 -0
  98. {supertable-2.0.0 → supertable-2.0.1}/supertable/locking/benchmarks/measure_lock_time.py +0 -0
  99. {supertable-2.0.0 → supertable-2.0.1}/supertable/locking/file_lock.py +0 -0
  100. {supertable-2.0.0 → supertable-2.0.1}/supertable/locking/redis_lock.py +0 -0
  101. {supertable-2.0.0 → supertable-2.0.1}/supertable/locking/tests/__init__.py +0 -0
  102. {supertable-2.0.0 → supertable-2.0.1}/supertable/locking/tests/test_file_lock.py +0 -0
  103. {supertable-2.0.0 → supertable-2.0.1}/supertable/locking/tests/test_redis_lock.py +0 -0
  104. {supertable-2.0.0 → supertable-2.0.1}/supertable/logging.py +0 -0
  105. {supertable-2.0.0 → supertable-2.0.1}/supertable/meta_reader.py +0 -0
  106. {supertable-2.0.0 → supertable-2.0.1}/supertable/mirroring/__init__.py +0 -0
  107. {supertable-2.0.0 → supertable-2.0.1}/supertable/mirroring/mirror_delta.py +0 -0
  108. {supertable-2.0.0 → supertable-2.0.1}/supertable/mirroring/mirror_formats.py +0 -0
  109. {supertable-2.0.0 → supertable-2.0.1}/supertable/mirroring/mirror_iceberg.py +0 -0
  110. {supertable-2.0.0 → supertable-2.0.1}/supertable/mirroring/mirror_parquet.py +0 -0
  111. {supertable-2.0.0 → supertable-2.0.1}/supertable/monitoring_writer.py +0 -0
  112. {supertable-2.0.0 → supertable-2.0.1}/supertable/plan_extender.py +0 -0
  113. {supertable-2.0.0 → supertable-2.0.1}/supertable/processing.py +0 -0
  114. {supertable-2.0.0 → supertable-2.0.1}/supertable/query_plan_manager.py +0 -0
  115. {supertable-2.0.0 → supertable-2.0.1}/supertable/rbac/__init__.py +0 -0
  116. {supertable-2.0.0 → supertable-2.0.1}/supertable/rbac/access_control.py +0 -0
  117. {supertable-2.0.0 → supertable-2.0.1}/supertable/rbac/filter_builder.py +0 -0
  118. {supertable-2.0.0 → supertable-2.0.1}/supertable/rbac/permissions.py +0 -0
  119. {supertable-2.0.0 → supertable-2.0.1}/supertable/rbac/role_manager.py +0 -0
  120. {supertable-2.0.0 → supertable-2.0.1}/supertable/rbac/row_column_security.py +0 -0
  121. {supertable-2.0.0 → supertable-2.0.1}/supertable/rbac/tests/test_filter_builder.py +0 -0
  122. {supertable-2.0.0 → supertable-2.0.1}/supertable/rbac/tests/test_rbac.py +0 -0
  123. {supertable-2.0.0 → supertable-2.0.1}/supertable/rbac/tests/test_rbac_per_table.py +0 -0
  124. {supertable-2.0.0 → supertable-2.0.1}/supertable/rbac/user_manager.py +0 -0
  125. {supertable-2.0.0 → supertable-2.0.1}/supertable/redis_catalog.py +0 -0
  126. {supertable-2.0.0 → supertable-2.0.1}/supertable/redis_connector.py +0 -0
  127. {supertable-2.0.0 → supertable-2.0.1}/supertable/service_registry.py +0 -0
  128. {supertable-2.0.0 → supertable-2.0.1}/supertable/simple_table.py +0 -0
  129. {supertable-2.0.0 → supertable-2.0.1}/supertable/staging_area.py +0 -0
  130. {supertable-2.0.0 → supertable-2.0.1}/supertable/storage/__init__.py +0 -0
  131. {supertable-2.0.0 → supertable-2.0.1}/supertable/storage/azure_storage.py +0 -0
  132. {supertable-2.0.0 → supertable-2.0.1}/supertable/storage/gcp_storage.py +0 -0
  133. {supertable-2.0.0 → supertable-2.0.1}/supertable/storage/local_storage.py +0 -0
  134. {supertable-2.0.0 → supertable-2.0.1}/supertable/storage/minio_storage.py +0 -0
  135. {supertable-2.0.0 → supertable-2.0.1}/supertable/storage/s3_storage.py +0 -0
  136. {supertable-2.0.0 → supertable-2.0.1}/supertable/storage/storage_factory.py +0 -0
  137. {supertable-2.0.0 → supertable-2.0.1}/supertable/storage/storage_interface.py +0 -0
  138. {supertable-2.0.0 → supertable-2.0.1}/supertable/storage/tests/test_storage.py +0 -0
  139. {supertable-2.0.0 → supertable-2.0.1}/supertable/super_pipe.py +0 -0
  140. {supertable-2.0.0 → supertable-2.0.1}/supertable/super_table.py +0 -0
  141. {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/__init__.py +0 -0
  142. {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_data_reader.py +0 -0
  143. {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_data_writer.py +0 -0
  144. {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_data_writer_comprehensive.py +0 -0
  145. {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_data_writer_tombstones.py +0 -0
  146. {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_dedup_on_read_write.py +0 -0
  147. {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_meta_reader.py +0 -0
  148. {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_newer_than.py +0 -0
  149. {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_process_delete_only.py +0 -0
  150. {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_processing.py +0 -0
  151. {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_query_sql.py +0 -0
  152. {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_simple_table.py +0 -0
  153. {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_small_file_compaction.py +0 -0
  154. {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_super_table.py +0 -0
  155. {supertable-2.0.0 → supertable-2.0.1}/supertable/tests/test_supertable_all.py +0 -0
  156. {supertable-2.0.0 → supertable-2.0.1}/supertable/utils/__init__.py +0 -0
  157. {supertable-2.0.0 → supertable-2.0.1}/supertable/utils/helper.py +0 -0
  158. {supertable-2.0.0 → supertable-2.0.1}/supertable/utils/sql_parser.py +0 -0
  159. {supertable-2.0.0 → supertable-2.0.1}/supertable/utils/tests/test_sql_parser_columns.py +0 -0
  160. {supertable-2.0.0 → supertable-2.0.1}/supertable/utils/timer.py +0 -0
  161. {supertable-2.0.0 → supertable-2.0.1}/supertable.egg-info/dependency_links.txt +0 -0
  162. {supertable-2.0.0 → supertable-2.0.1}/supertable.egg-info/entry_points.txt +0 -0
  163. {supertable-2.0.0 → supertable-2.0.1}/supertable.egg-info/requires.txt +0 -0
  164. {supertable-2.0.0 → supertable-2.0.1}/supertable.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: supertable
3
- Version: 2.0.0
3
+ Version: 2.0.1
4
4
  Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
5
5
  Author: Levente Kupas
6
6
  Author-email: Levente Kupas <lkupas@kladnasoft.com>
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "supertable"
7
- version = "2.0.0"
7
+ version = "2.0.1"
8
8
  description = "SuperTable — versioned data lake library for SQL analytics on Parquet + Redis."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
19
19
 
20
20
  setup(
21
21
  name="supertable",
22
- version="2.0.0",
22
+ version="2.0.1",
23
23
  description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
24
24
  long_description=long_description,
25
25
  long_description_content_type="text/markdown",
@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
25
25
  project documentation for the full API surface.
26
26
  """
27
27
 
28
- __version__ = "2.0.0"
28
+ __version__ = "2.0.1"
29
29
 
30
30
  # Re-export the core public surface so users can do ``from supertable import …``
31
31
  # instead of remembering submodule paths.
@@ -21,7 +21,7 @@ logger = logging.getLogger(__name__)
21
21
  def create_consumer(organization: str, group_name: str, start_from: str = "$") -> Dict[str, Any]:
22
22
  """Create an external SIEM consumer group on the audit stream."""
23
23
  try:
24
- from supertable.server_common import redis_client
24
+ from supertable.redis_infra import redis_client
25
25
  from supertable.audit.writer_redis import RedisAuditWriter
26
26
  writer = RedisAuditWriter(redis_client, organization, "", maxlen=0)
27
27
  ok = writer.create_consumer_group(group_name, start_from)
@@ -34,7 +34,7 @@ def create_consumer(organization: str, group_name: str, start_from: str = "$") -
34
34
  def delete_consumer(organization: str, group_name: str) -> Dict[str, Any]:
35
35
  """Remove an external SIEM consumer group."""
36
36
  try:
37
- from supertable.server_common import redis_client
37
+ from supertable.redis_infra import redis_client
38
38
  from supertable.audit.writer_redis import RedisAuditWriter
39
39
  writer = RedisAuditWriter(redis_client, organization, "", maxlen=0)
40
40
  ok = writer.delete_consumer_group(group_name)
@@ -47,7 +47,7 @@ def delete_consumer(organization: str, group_name: str) -> Dict[str, Any]:
47
47
  def list_consumers(organization: str) -> List[Dict[str, Any]]:
48
48
  """List all consumer groups with lag info."""
49
49
  try:
50
- from supertable.server_common import redis_client
50
+ from supertable.redis_infra import redis_client
51
51
  from supertable.audit.writer_redis import RedisAuditWriter
52
52
  writer = RedisAuditWriter(redis_client, organization, "", maxlen=0)
53
53
  return writer.list_consumer_groups()
@@ -132,7 +132,7 @@ class AuditLogger:
132
132
  def _init_redis(self) -> None:
133
133
  """Initialize Redis Stream writer."""
134
134
  try:
135
- from supertable.server_common import redis_client
135
+ from supertable.redis_infra import redis_client
136
136
  from supertable.audit.writer_redis import RedisAuditWriter
137
137
  self._redis_writer = RedisAuditWriter(
138
138
  redis_client=redis_client,
@@ -93,7 +93,7 @@ def _query_redis(
93
93
  ) -> List[Dict[str, Any]]:
94
94
  """Query recent events from the Redis Stream."""
95
95
  try:
96
- from supertable.server_common import redis_client
96
+ from supertable.redis_infra import redis_client
97
97
  from supertable.audit.writer_redis import RedisAuditWriter
98
98
  writer = RedisAuditWriter(redis_client, organization, "", maxlen=0)
99
99
  return writer.query(start_ms=start_ms, end_ms=end_ms, count=limit)
@@ -86,7 +86,7 @@ def is_legal_hold_active(organization: str) -> bool:
86
86
  """
87
87
  # Try Redis first (runtime override)
88
88
  try:
89
- from supertable.server_common import redis_client
89
+ from supertable.redis_infra import redis_client
90
90
  val = redis_client.get(_legal_hold_key(organization))
91
91
  if val is not None:
92
92
  decoded = val if isinstance(val, str) else val.decode("utf-8")
@@ -128,7 +128,7 @@ def set_legal_hold(enabled: bool, organization: str = "") -> Dict[str, Any]:
128
128
 
129
129
  # Persist to Redis
130
130
  try:
131
- from supertable.server_common import redis_client
131
+ from supertable.redis_infra import redis_client
132
132
  redis_client.set(_legal_hold_key(organization), "1" if enabled else "0")
133
133
  except Exception as e:
134
134
  logger.error("[audit-retention] Failed to persist legal hold to Redis: %s", e)
@@ -83,7 +83,7 @@ class TestIsLegalHoldActive:
83
83
  ) -> None:
84
84
  fake_client = _make_redis_client(get_value=redis_value)
85
85
  fake_module = SimpleNamespace(redis_client=fake_client)
86
- monkeypatch.setitem(__import__("sys").modules, "supertable.server_common", fake_module)
86
+ monkeypatch.setitem(__import__("sys").modules, "supertable.redis_infra", fake_module)
87
87
  assert retention.is_legal_hold_active("acme") is True
88
88
 
89
89
  @pytest.mark.parametrize("redis_value", [b"0", "false", b"no"])
@@ -92,7 +92,7 @@ class TestIsLegalHoldActive:
92
92
  ) -> None:
93
93
  fake_client = _make_redis_client(get_value=redis_value)
94
94
  fake_module = SimpleNamespace(redis_client=fake_client)
95
- monkeypatch.setitem(__import__("sys").modules, "supertable.server_common", fake_module)
95
+ monkeypatch.setitem(__import__("sys").modules, "supertable.redis_infra", fake_module)
96
96
  assert retention.is_legal_hold_active("acme") is False
97
97
 
98
98
  def test_falls_back_to_settings_when_redis_returns_none(
@@ -102,7 +102,7 @@ class TestIsLegalHoldActive:
102
102
  fake_server_common = SimpleNamespace(redis_client=fake_client)
103
103
  monkeypatch.setitem(
104
104
  __import__("sys").modules,
105
- "supertable.server_common",
105
+ "supertable.redis_infra",
106
106
  fake_server_common,
107
107
  )
108
108
 
@@ -125,7 +125,7 @@ class TestIsLegalHoldActive:
125
125
  raise RuntimeError("redis unreachable")
126
126
 
127
127
  monkeypatch.setitem(
128
- __import__("sys").modules, "supertable.server_common", BoomModule()
128
+ __import__("sys").modules, "supertable.redis_infra", BoomModule()
129
129
  )
130
130
 
131
131
  # Make settings import fail by removing the attribute
@@ -155,7 +155,7 @@ class TestSetLegalHold:
155
155
  ) -> None:
156
156
  fake_client = _make_redis_client()
157
157
  fake_module = SimpleNamespace(redis_client=fake_client)
158
- monkeypatch.setitem(__import__("sys").modules, "supertable.server_common", fake_module)
158
+ monkeypatch.setitem(__import__("sys").modules, "supertable.redis_infra", fake_module)
159
159
 
160
160
  # Stub out the audit emit chain so the test never touches the real logger.
161
161
  import supertable.audit as audit_pkg
@@ -181,7 +181,7 @@ class TestSetLegalHold:
181
181
  fake_client = MagicMock()
182
182
  fake_client.set.side_effect = RuntimeError("redis down")
183
183
  fake_module = SimpleNamespace(redis_client=fake_client)
184
- monkeypatch.setitem(__import__("sys").modules, "supertable.server_common", fake_module)
184
+ monkeypatch.setitem(__import__("sys").modules, "supertable.redis_infra", fake_module)
185
185
 
186
186
  result = retention.set_legal_hold(False, organization="acme")
187
187
  assert result["ok"] is False
@@ -0,0 +1,467 @@
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+ import json
5
+ from typing import Dict, Iterator, List, Optional, Tuple
6
+ from pathlib import Path
7
+ from urllib.parse import urlparse
8
+ import time
9
+
10
+
11
+ import redis
12
+ from redis import Sentinel
13
+ from redis.sentinel import MasterNotFoundError
14
+
15
+
16
+
17
+ from supertable.config.settings import settings as _cfg
18
+
19
+ logger = logging.getLogger(__name__)
20
+ # ------------------------------ Settings ------------------------------
21
+
22
+ class Settings:
23
+ """Thin adapter that delegates to the central config.settings singleton.
24
+
25
+ Attributes that config.settings does not carry (e.g. TEMPLATES_DIR with
26
+ a path derived from __file__) are resolved here.
27
+ """
28
+
29
+ def __init__(self) -> None:
30
+ self.SUPERTABLE_ORGANIZATION: str = _cfg.SUPERTABLE_ORGANIZATION
31
+ self.SUPERTABLE_SUPERUSER_TOKEN: str = _cfg.SUPERTABLE_SUPERUSER_TOKEN
32
+ self.SUPERTABLE_SESSION_SECRET: str = _cfg.SUPERTABLE_SESSION_SECRET
33
+
34
+ self.SUPERTABLE_REDIS_URL: Optional[str] = _cfg.SUPERTABLE_REDIS_URL or None
35
+ self.SUPERTABLE_REDIS_HOST: str = _cfg.SUPERTABLE_REDIS_HOST
36
+ self.SUPERTABLE_REDIS_PORT: int = _cfg.SUPERTABLE_REDIS_PORT
37
+ self.SUPERTABLE_REDIS_DB: int = _cfg.SUPERTABLE_REDIS_DB
38
+ self.SUPERTABLE_REDIS_PASSWORD: Optional[str] = _cfg.SUPERTABLE_REDIS_PASSWORD or None
39
+ self.SUPERTABLE_REDIS_USERNAME: Optional[str] = _cfg.SUPERTABLE_REDIS_USERNAME or None
40
+
41
+ self.SUPERTABLE_REDIS_SENTINEL: Optional[str] = str(_cfg.SUPERTABLE_REDIS_SENTINEL) if _cfg.SUPERTABLE_REDIS_SENTINEL else None
42
+ self.SUPERTABLE_REDIS_SENTINELS: Optional[str] = _cfg.SUPERTABLE_REDIS_SENTINELS or None
43
+ self.SUPERTABLE_REDIS_SENTINEL_MASTER: Optional[str] = _cfg.SUPERTABLE_REDIS_SENTINEL_MASTER or None
44
+ self.SUPERTABLE_REDIS_SENTINEL_PASSWORD: Optional[str] = _cfg.SUPERTABLE_REDIS_SENTINEL_PASSWORD or None
45
+ self.SUPERTABLE_REDIS_SENTINEL_STRICT: Optional[str] = _cfg.SUPERTABLE_REDIS_SENTINEL_STRICT or None
46
+
47
+ self.SUPERTABLE_LOGIN_MASK: int = _cfg.SUPERTABLE_LOGIN_MASK
48
+
49
+ self.DOTENV_PATH: str = _cfg.DOTENV_PATH
50
+
51
+ # TEMPLATES_DIR: use central setting if provided, otherwise derive from __file__
52
+ self.TEMPLATES_DIR: str = (
53
+ _cfg.TEMPLATES_DIR
54
+ or str(Path(__file__).resolve().parent / "webui" / "templates")
55
+ )
56
+
57
+ self.SECURE_COOKIES: bool = _cfg.SECURE_COOKIES
58
+
59
+
60
+ settings = Settings()
61
+ if settings.SUPERTABLE_LOGIN_MASK not in (1, 2, 3):
62
+ raise RuntimeError(
63
+ f"Invalid SUPERTABLE_LOGIN_MASK (must be 1, 2, or 3): {settings.SUPERTABLE_LOGIN_MASK}"
64
+ )
65
+
66
+ _missing_envs: List[str] = []
67
+ if not settings.SUPERTABLE_ORGANIZATION:
68
+ _missing_envs.append("SUPERTABLE_ORGANIZATION")
69
+ if not (settings.SUPERTABLE_SUPERUSER_TOKEN or "").strip():
70
+ _missing_envs.append("SUPERTABLE_SUPERUSER_TOKEN")
71
+ if _missing_envs:
72
+ raise RuntimeError("Missing required environment variables: " + ", ".join(_missing_envs))
73
+
74
+
75
+ def _now_ms() -> int:
76
+ from time import time as _t
77
+ return int(_t() * 1000)
78
+
79
+
80
+ # ------------------------------ Catalog (import or fallback) ------------------------------
81
+
82
+ def _root_key(org: str, sup: str) -> str:
83
+ return f"supertable:{org}:{sup}:meta:root"
84
+
85
+
86
+ def _leaf_key(org: str, sup: str, simple: str) -> str:
87
+ return f"supertable:{org}:{sup}:meta:leaf:{simple}"
88
+
89
+
90
+ def _mirrors_key(org: str, sup: str) -> str:
91
+ return f"supertable:{org}:{sup}:meta:mirrors"
92
+
93
+
94
+ class _FallbackCatalog:
95
+ def __init__(self, r: redis.Redis):
96
+ self.r = r
97
+
98
+ def ensure_root(self, org: str, sup: str) -> None:
99
+ key = _root_key(org, sup)
100
+ if not self.r.exists(key):
101
+ self.r.set(key, json.dumps({"version": 0, "ts": _now_ms()}))
102
+
103
+ def get_root(self, org: str, sup: str) -> Optional[Dict]:
104
+ raw = self.r.get(_root_key(org, sup))
105
+ return json.loads(raw) if raw else None
106
+
107
+ def get_leaf(self, org: str, sup: str, simple: str) -> Optional[Dict]:
108
+ raw = self.r.get(_leaf_key(org, sup, simple))
109
+ return json.loads(raw) if raw else None
110
+
111
+ def get_mirrors(self, org: str, sup: str) -> List[str]:
112
+ raw = self.r.get(_mirrors_key(org, sup))
113
+ if not raw:
114
+ return []
115
+ try:
116
+ obj = json.loads(raw)
117
+ except Exception:
118
+ return []
119
+ out = []
120
+ for f in (obj.get("formats") or []):
121
+ fu = str(f).upper()
122
+ if fu in ("DELTA", "ICEBERG", "PARQUET") and fu not in out:
123
+ out.append(fu)
124
+ return out
125
+
126
+ def set_mirrors(self, org: str, sup: str, formats: List[str]) -> List[str]:
127
+ uniq = []
128
+ for f in (formats or []):
129
+ fu = str(f).upper()
130
+ if fu in ("DELTA", "ICEBERG", "PARQUET") and fu not in uniq:
131
+ uniq.append(fu)
132
+ self.r.set(_mirrors_key(org, sup), json.dumps({"formats": uniq, "ts": _now_ms()}))
133
+ return uniq
134
+
135
+ def enable_mirror(self, org: str, sup: str, fmt: str) -> List[str]:
136
+ cur = self.get_mirrors(org, sup)
137
+ fu = str(fmt).upper()
138
+ if fu not in ("DELTA", "ICEBERG", "PARQUET") or fu in cur:
139
+ return cur
140
+ return self.set_mirrors(org, sup, cur + [fu])
141
+
142
+ def disable_mirror(self, org: str, sup: str, fmt: str) -> List[str]:
143
+ cur = self.get_mirrors(org, sup)
144
+ fu = str(fmt).upper()
145
+ nxt = [x for x in cur if x != fu]
146
+ return self.set_mirrors(org, sup, nxt)
147
+
148
+ def scan_leaf_keys(self, org: str, sup: str, count: int = 1000) -> Iterator[str]:
149
+ pattern = f"supertable:{org}:{sup}:meta:leaf:*"
150
+ cursor = 0
151
+ while True:
152
+ cursor, keys = self.r.scan(cursor=cursor, match=pattern, count=max(1, int(count)))
153
+ for k in keys:
154
+ yield k if isinstance(k, str) else k.decode("utf-8")
155
+ if cursor == 0:
156
+ break
157
+
158
+ def scan_leaf_items(self, org: str, sup: str, count: int = 1000) -> Iterator[Dict]:
159
+ batch: List[str] = []
160
+ for key in self.scan_leaf_keys(org, sup, count=count):
161
+ batch.append(key)
162
+ if len(batch) >= count:
163
+ yield from self._fetch_batch(batch)
164
+ batch = []
165
+ if batch:
166
+ yield from self._fetch_batch(batch)
167
+
168
+ def _fetch_batch(self, keys: List[str]) -> Iterator[Dict]:
169
+ pipe = self.r.pipeline()
170
+ for k in keys:
171
+ pipe.get(k)
172
+ vals = pipe.execute()
173
+ for k, raw in zip(keys, vals):
174
+ if not raw:
175
+ continue
176
+ try:
177
+ obj = json.loads(raw if isinstance(raw, str) else raw.decode("utf-8"))
178
+ simple = k.rsplit("meta:leaf:", 1)[-1]
179
+ yield {
180
+ "simple": simple,
181
+ "version": int(obj.get("version", -1)),
182
+ "ts": int(obj.get("ts", 0)),
183
+ "path": obj.get("path", ""),
184
+ }
185
+ except Exception:
186
+ continue
187
+
188
+ # -- RBAC methods (mirrors RedisCatalog API using correct rbac: key namespace) --
189
+
190
+ @staticmethod
191
+ def _decode_member(m) -> str:
192
+ return m if isinstance(m, str) else m.decode("utf-8")
193
+
194
+ def get_users(self, org: str, sup: str) -> List[Dict]:
195
+ users: List[Dict] = []
196
+ try:
197
+ index_key = f"supertable:{org}:{sup}:rbac:users:index"
198
+ members = self.r.smembers(index_key)
199
+ for uid_raw in (members or []):
200
+ uid = self._decode_member(uid_raw)
201
+ doc_key = f"supertable:{org}:{sup}:rbac:users:doc:{uid}"
202
+ raw = self.r.hgetall(doc_key)
203
+ if raw:
204
+ data: Dict = dict(raw)
205
+ data.setdefault("user_id", uid)
206
+ data.setdefault("hash", uid)
207
+ if "roles" in data:
208
+ try:
209
+ data["roles"] = json.loads(data["roles"])
210
+ except (json.JSONDecodeError, TypeError):
211
+ data["roles"] = []
212
+ users.append(data)
213
+ except Exception as e:
214
+ logger.warning("_FallbackCatalog.get_users error: %s", e)
215
+ return users
216
+
217
+ def get_roles(self, org: str, sup: str) -> List[Dict]:
218
+ roles: List[Dict] = []
219
+ try:
220
+ index_key = f"supertable:{org}:{sup}:rbac:roles:index"
221
+ members = self.r.smembers(index_key)
222
+ for rid_raw in (members or []):
223
+ rid = self._decode_member(rid_raw)
224
+ doc_key = f"supertable:{org}:{sup}:rbac:roles:doc:{rid}"
225
+ raw = self.r.hgetall(doc_key)
226
+ if raw:
227
+ data: Dict = dict(raw)
228
+ data.setdefault("role_id", rid)
229
+ data.setdefault("hash", rid)
230
+ for field in ("tables", "columns", "filters"):
231
+ if field in data:
232
+ try:
233
+ data[field] = json.loads(data[field])
234
+ except (json.JSONDecodeError, TypeError):
235
+ pass
236
+ roles.append(data)
237
+ except Exception as e:
238
+ logger.warning("_FallbackCatalog.get_roles error: %s", e)
239
+ return roles
240
+
241
+ def get_user_details(self, org: str, sup: str, user_id: str) -> Optional[Dict]:
242
+ try:
243
+ doc_key = f"supertable:{org}:{sup}:rbac:users:doc:{user_id}"
244
+ raw = self.r.hgetall(doc_key)
245
+ if not raw:
246
+ return None
247
+ data: Dict = dict(raw)
248
+ if "roles" in data:
249
+ try:
250
+ data["roles"] = json.loads(data["roles"])
251
+ except (json.JSONDecodeError, TypeError):
252
+ data["roles"] = []
253
+ return data
254
+ except Exception as e:
255
+ logger.warning("_FallbackCatalog.get_user_details error: %s", e)
256
+ return None
257
+
258
+ def get_role_details(self, org: str, sup: str, role_id: str) -> Optional[Dict]:
259
+ try:
260
+ doc_key = f"supertable:{org}:{sup}:rbac:roles:doc:{role_id}"
261
+ raw = self.r.hgetall(doc_key)
262
+ if not raw:
263
+ return None
264
+ data: Dict = dict(raw)
265
+ for field in ("tables", "columns", "filters"):
266
+ if field in data:
267
+ try:
268
+ data[field] = json.loads(data[field])
269
+ except (json.JSONDecodeError, TypeError):
270
+ pass
271
+ return data
272
+ except Exception as e:
273
+ logger.warning("_FallbackCatalog.get_role_details error: %s", e)
274
+ return None
275
+
276
+ def rbac_get_user_id_by_username(self, org: str, sup: str, username: str) -> Optional[str]:
277
+ try:
278
+ name_map_key = f"supertable:{org}:{sup}:rbac:users:name_to_id"
279
+ val = self.r.hget(name_map_key, username.lower())
280
+ if val is None:
281
+ return None
282
+ return self._decode_member(val)
283
+ except Exception as e:
284
+ logger.warning("_FallbackCatalog.rbac_get_user_id_by_username error: %s", e)
285
+ return None
286
+
287
+
288
+ def _coerce_password(pw: Optional[str]) -> Optional[str]:
289
+ if pw is None:
290
+ return None
291
+ v = pw.strip()
292
+ # Treat these as "no password"
293
+ if v in ("", "None", "none", "null", "NULL"):
294
+ return None
295
+ return v
296
+
297
+
298
+ def _build_redis_client() -> redis.Redis:
299
+ """
300
+ Build a Redis client from SUPERTABLE_* envs.
301
+
302
+ Precedence:
303
+ 1) SUPERTABLE_REDIS_URL (parsed)
304
+ 2) SUPERTABLE_REDIS_HOST/PORT/DB/PASSWORD (overrides URL parts if provided)
305
+
306
+ Sentinel:
307
+ If SUPERTABLE_REDIS_SENTINEL is enabled and SUPERTABLE_REDIS_SENTINELS is set,
308
+ use the same Sentinel connection behavior as RedisCatalog (ping-probe + optional fallback).
309
+ """
310
+ settings = Settings()
311
+ url = (settings.SUPERTABLE_REDIS_URL or "").strip() or None
312
+
313
+ host = settings.SUPERTABLE_REDIS_HOST
314
+ port = settings.SUPERTABLE_REDIS_PORT
315
+ db = settings.SUPERTABLE_REDIS_DB
316
+ username = (settings.SUPERTABLE_REDIS_USERNAME or "").strip() or None
317
+ password = _coerce_password(settings.SUPERTABLE_REDIS_PASSWORD)
318
+
319
+ use_ssl = url.startswith("rediss://") if url else False
320
+
321
+ if url:
322
+ u = urlparse(url)
323
+ if u.scheme not in ("redis", "rediss"):
324
+ raise ValueError(f"Unsupported Redis URL scheme: {u.scheme}")
325
+ # Extract from URL
326
+ if u.hostname:
327
+ host = u.hostname
328
+ if u.port:
329
+ port = u.port
330
+ # db from path: "/0", "/1", ...
331
+ if u.path and len(u.path) > 1:
332
+ try:
333
+ db_from_url = int(u.path.lstrip("/"))
334
+ db = db_from_url
335
+ except Exception:
336
+ pass
337
+ if u.username:
338
+ username = u.username
339
+ if u.password:
340
+ password = _coerce_password(u.password)
341
+
342
+ # Sentinel detection + options
343
+ sentinel_enabled = (settings.SUPERTABLE_REDIS_SENTINEL or "").strip().lower() in (
344
+ "1",
345
+ "true",
346
+ "yes",
347
+ "y",
348
+ "on",
349
+ )
350
+ sentinel_strict = (settings.SUPERTABLE_REDIS_SENTINEL_STRICT or "").strip().lower() in (
351
+ "1",
352
+ "true",
353
+ "yes",
354
+ "y",
355
+ "on",
356
+ )
357
+ sentinel_master = (settings.SUPERTABLE_REDIS_SENTINEL_MASTER or "").strip() or "mymaster"
358
+
359
+ sentinel_password = _coerce_password(settings.SUPERTABLE_REDIS_SENTINEL_PASSWORD) or password
360
+
361
+ # Single-password setup: if Sentinel is enabled and Redis password is not set, reuse Sentinel password.
362
+ if sentinel_enabled and password is None and sentinel_password:
363
+ password = sentinel_password
364
+
365
+ # If username is set but password is None, drop username (ACL requires both)
366
+ if username and not password:
367
+ username = None
368
+
369
+ sentinel_hosts: List[Tuple[str, int]] = []
370
+ sentinel_raw = (settings.SUPERTABLE_REDIS_SENTINELS or "").strip()
371
+ if sentinel_raw:
372
+ for part in sentinel_raw.split(","):
373
+ part = part.strip()
374
+ if not part:
375
+ continue
376
+ try:
377
+ h, p = part.split(":")
378
+ sentinel_hosts.append((h.strip(), int(p)))
379
+ except ValueError:
380
+ # Keep behavior non-fatal; invalid entries are ignored.
381
+ continue
382
+
383
+ if sentinel_enabled and sentinel_hosts:
384
+ sentinel_kwargs: dict = {
385
+ "socket_timeout": 0.5,
386
+ "decode_responses": True,
387
+ "ssl": use_ssl,
388
+ }
389
+ if sentinel_password:
390
+ sentinel_kwargs["password"] = sentinel_password
391
+ if username:
392
+ sentinel_kwargs["username"] = username
393
+
394
+ sentinel = Sentinel(
395
+ sentinel_hosts,
396
+ sentinel_kwargs=sentinel_kwargs,
397
+ socket_timeout=0.5,
398
+ decode_responses=True,
399
+ ssl=use_ssl,
400
+ username=username,
401
+ password=password,
402
+ )
403
+
404
+ sentinel_client: redis.Redis = sentinel.master_for(
405
+ sentinel_master,
406
+ db=db,
407
+ decode_responses=True,
408
+ ssl=use_ssl,
409
+ username=username,
410
+ password=password,
411
+ )
412
+
413
+ # Fail-fast probe (matches RedisCatalog behavior).
414
+ sentinel_err: Optional[BaseException] = None
415
+ deadline = time.time() + 3.0
416
+ while time.time() < deadline:
417
+ try:
418
+ sentinel_client.ping()
419
+ sentinel_err = None
420
+ break
421
+ except (MasterNotFoundError, redis.RedisError, OSError) as e:
422
+ sentinel_err = e
423
+ time.sleep(0.2)
424
+
425
+ if sentinel_err is None:
426
+ return sentinel_client
427
+
428
+ if sentinel_strict:
429
+ raise sentinel_err
430
+
431
+ # Non-strict fallback to standard Redis
432
+ return redis.Redis(
433
+ host=host,
434
+ port=port,
435
+ db=db,
436
+ password=password,
437
+ username=username,
438
+ decode_responses=True,
439
+ ssl=use_ssl,
440
+ )
441
+
442
+ # Standard Redis mode
443
+ return redis.Redis(
444
+ host=host,
445
+ port=port,
446
+ db=db,
447
+ password=password,
448
+ username=username,
449
+ decode_responses=True,
450
+ ssl=use_ssl,
451
+ )
452
+
453
+
454
+ def _build_catalog() -> Tuple[object, redis.Redis]:
455
+ r = _build_redis_client()
456
+ try:
457
+ from supertable.redis_catalog import RedisCatalog as _RC # type: ignore
458
+ return _RC(), r
459
+ except Exception:
460
+ try:
461
+ from redis_catalog import RedisCatalog as _RC # type: ignore
462
+ return _RC(), r
463
+ except Exception:
464
+ return _FallbackCatalog(r), r
465
+
466
+
467
+ catalog, redis_client = _build_catalog()
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: supertable
3
- Version: 2.0.0
3
+ Version: 2.0.1
4
4
  Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
5
5
  Author: Levente Kupas
6
6
  Author-email: Levente Kupas <lkupas@kladnasoft.com>
@@ -15,7 +15,7 @@ supertable/processing.py
15
15
  supertable/query_plan_manager.py
16
16
  supertable/redis_catalog.py
17
17
  supertable/redis_connector.py
18
- supertable/server_common.py
18
+ supertable/redis_infra.py
19
19
  supertable/service_registry.py
20
20
  supertable/simple_table.py
21
21
  supertable/staging_area.py